Implement an API for proxy and hostname configuration, and centralize management of config.json

Change-Type: minor
Signed-off-by: Pablo Carranza Velez <pablo@resin.io>

gosuper/gosuper/api.go

@@ -38,6 +38,10 @@ type LogToDisplayBody struct {
 	Enable bool
 }
 
+type RestartServiceBody struct {
+	Name string
+}
+
 func jsonResponse(writer http.ResponseWriter, response interface{}, status int) {
 	jsonBody, err := json.Marshal(response)
 	if err != nil {
@@ -278,3 +282,29 @@ func LogToDisplayControl(writer http.ResponseWriter, request *http.Request) {
 		sendResponse(true, "", http.StatusOK)
 	}
 }
+
+//RestartService is used to restart a systemd service by name
+func RestartService(writer http.ResponseWriter, request *http.Request) {
+	sendResponse, sendError := responseSenders(writer)
+	var body RestartServiceBody
+	if err := parseJSONBody(&body, request); err != nil {
+		log.Println(err)
+		sendResponse("Error", err.Error(), http.StatusBadRequest)
+		return
+	}
+
+	if systemd.Dbus == nil {
+		sendError(fmt.Errorf("Systemd dbus unavailable, cannot restart service."))
+		return
+	}
+
+	actionDescr := "Service Restart"
+
+	if _, err := systemd.Dbus.RestartUnit(body.Name+".service", "fail", nil); err != nil {
+		log.Printf("%s: %s\n", actionDescr, err)
+		sendError(err)
+		return
+	}
+	log.Printf("%sd\n", actionDescr)
+	sendResponse("OK", "", http.StatusOK)
+}

gosuper/gosuper/main.go

@@ -26,6 +26,7 @@ func setupApi(router *mux.Router) {
 	apiv1.HandleFunc("/shutdown", ShutdownHandler).Methods("POST")
 	apiv1.HandleFunc("/vpncontrol", VPNControl).Methods("POST")
 	apiv1.HandleFunc("/set-log-to-display", LogToDisplayControl).Methods("POST")
+	apiv1.HandleFunc("/restart-service", RestartService).Methods("POST")
 }
 
 func startApi(listenAddress string, router *mux.Router) {

package.json

@@ -40,6 +40,7 @@
     "log-timestamp": "^0.1.2",
     "memoizee": "^0.4.1",
     "mixpanel": "0.0.20",
+    "mkdirp": "^0.5.1",
     "network-checker": "~0.0.5",
     "node-loader": "^0.6.0",
     "null-loader": "^0.1.1",
@@ -59,4 +60,4 @@
     "versionist": "^2.8.0",
     "webpack": "^3.0.0"
   }
-}
+}

src/api.coffee

@@ -7,6 +7,7 @@ config = require './config'
 device = require './device'
 _ = require 'lodash'
 proxyvisor = require './proxyvisor'
+hostConfig = require './host-config'
 
 module.exports = (application) ->
 	authenticate = (req, res, next) ->
@@ -208,7 +209,6 @@ module.exports = (application) ->
 		.catch (err) ->
 			res.status(503).send(err?.message or err or 'Unknown error')
 
-
 	# Expires the supervisor's API key and generates a new one.
 	# It also communicates the new key to the Resin API.
 	unparsedRouter.post '/v1/regenerate-api-key', (req, res) ->
@@ -219,6 +219,20 @@ module.exports = (application) ->
 		.catch (err) ->
 			res.status(503).send(err?.message or err or 'Unknown error')
 
+	parsedRouter.get '/v1/device/host-config', (req, res) ->
+		hostConfig.get()
+		.then (conf) ->
+			res.json(conf)
+		.catch (err) ->
+			res.status(503).send(err?.message or err or 'Unknown error')
+
+	parsedRouter.patch '/v1/device/host-config', (req, res) ->
+		hostConfig.patch(req.body)
+		.then ->
+			res.status(200).send('OK')
+		.catch (err) ->
+			res.status(503).send(err?.message or err or 'Unknown error')
+
 	unparsedRouter.get '/v1/device', (req, res) ->
 		res.json(device.getState())
 

src/bootstrap.coffee

@@ -5,7 +5,6 @@ deviceRegister = require 'resin-register-device'
 { resinApi, request } = require './request'
 fs = Promise.promisifyAll(require('fs'))
 config = require './config'
-configPath = '/boot/config.json'
 appsPath  = '/boot/apps.json'
 _ = require 'lodash'
 deviceConfig = require './device-config'
@@ -13,23 +12,13 @@ TypedError = require 'typed-error'
 osRelease = require './lib/os-release'
 semver = require 'semver'
 semverRegex = require('semver-regex')
-
+configJson = require './config-json'
-userConfig = {}
 
 DuplicateUuidError = (err) -> _.startsWith(err.message, '"uuid" must be unique')
 exports.ExchangeKeyError = class ExchangeKeyError extends TypedError
 
 bootstrapper = {}
 
-writeAndSyncFile = (path, data) ->
-	fs.openAsync(path, 'w')
-	.then (fd) ->
-		fs.writeAsync(fd, data, 0, 'utf8')
-		.then ->
-			fs.fsyncAsync(fd)
-		.then ->
-			fs.closeAsync(fd)
-
 loadPreloadedApps = ->
 	devConfig = {}
 	knex('app').select()
@@ -37,27 +26,29 @@ loadPreloadedApps = ->
 		if apps.length > 0
 			console.log('Preloaded apps already loaded, skipping')
 			return
-		fs.readFileAsync(appsPath, 'utf8')
+		configJson.getAll()
-		.then(JSON.parse)
+		.then (userConfig) ->
-		.map (app) ->
+			fs.readFileAsync(appsPath, 'utf8')
-			utils.extendEnvVars(app.env, userConfig.uuid, userConfig.deviceApiKey, app.appId, app.name, app.commit)
+			.then(JSON.parse)
-			.then (extendedEnv) ->
+			.map (app) ->
-				app.env = JSON.stringify(extendedEnv)
+				utils.extendEnvVars(app.env, userConfig.uuid, userConfig.deviceApiKey, app.appId, app.name, app.commit)
-				app.markedForDeletion = false
+				.then (extendedEnv) ->
-				_.merge(devConfig, app.config)
+					app.env = JSON.stringify(extendedEnv)
-				app.config = JSON.stringify(app.config)
+					app.markedForDeletion = false
-				knex('app').insert(app)
+					_.merge(devConfig, app.config)
-		.then ->
+					app.config = JSON.stringify(app.config)
-			deviceConfig.set({ targetValues: devConfig })
+					knex('app').insert(app)
+			.then ->
+				deviceConfig.set({ targetValues: devConfig })
 		.catch (err) ->
 			utils.mixpanelTrack('Loading preloaded apps failed', { error: err })
 
-fetchDevice = (apiKey) ->
+fetchDevice = (uuid, apiKey) ->
 	resinApi.get
 		resource: 'device'
 		options:
 			filter:
-				uuid: userConfig.uuid
+				uuid: uuid
 		customOptions:
 			apikey: apiKey
 	.get(0)
@@ -65,33 +56,47 @@ fetchDevice = (apiKey) ->
 	.timeout(config.apiTimeout)
 
 exchangeKey = ->
-	Promise.try ->
+	configJson.getAll()
-		# If we have an existing device key we first check if it's valid, because if it is we can just use that
+	.then (userConfig) ->
-		if userConfig.deviceApiKey?
+		Promise.try ->
-			fetchDevice(userConfig.deviceApiKey)
+			# If we have an existing device key we first check if it's valid, because if it is we can just use that
-	.then (device) ->
+			if userConfig.deviceApiKey?
-		if device?
+				fetchDevice(userConfig.uuid, userConfig.deviceApiKey)
-			return device
-		# If it's not valid/doesn't exist then we try to use the user/provisioning api key for the exchange
-		fetchDevice(userConfig.apiKey)
 		.then (device) ->
-			if not device?
+			if device?
-				throw new ExchangeKeyError("Couldn't fetch device with provisioning key")
+				return device
-			# We found the device, we can try to register a working device key for it
+			# If it's not valid/doesn't exist then we try to use the user/provisioning api key for the exchange
-			userConfig.deviceApiKey ?= deviceRegister.generateUniqueKey()
+			fetchDevice(userConfig.uuid, userConfig.apiKey)
-			request.postAsync("#{config.apiEndpoint}/api-key/device/#{device.id}/device-key?apikey=#{userConfig.apiKey}", {
+			.then (device) ->
-				json: true
+				if not device?
-				body:
+					throw new ExchangeKeyError("Couldn't fetch device with provisioning key")
-					apiKey: userConfig.deviceApiKey
+				# We found the device, we can try to register a working device key for it
-			})
+				Promise.try ->
-			.spread (res, body) ->
+					if !userConfig.deviceApiKey?
-				if res.statusCode != 200
+						deviceApiKey = deviceRegister.generateUniqueKey()
-					throw new ExchangeKeyError("Couldn't register device key with provisioning key")
+						configJson.set({ deviceApiKey })
-			.return(device)
+						.return(deviceApiKey)
+					else
+						return userConfig.deviceApiKey
+				.then (deviceApiKey) ->
+					request.postAsync("#{config.apiEndpoint}/api-key/device/#{device.id}/device-key?apikey=#{userConfig.apiKey}", {
+						json: true
+						body:
+							apiKey: deviceApiKey
+					})
+				.spread (res, body) ->
+					if res.statusCode != 200
+						throw new ExchangeKeyError("Couldn't register device key with provisioning key")
+				.return(device)
 
 bootstrap = ->
-	Promise.try ->
+	configJson.get('deviceType')
-		userConfig.deviceType ?= 'raspberry-pi'
+	.then (deviceType) ->
+		if !deviceType?
+			configJson.set(deviceType: 'raspberry-pi')
+	.then ->
+		configJson.getAll()
+	.then (userConfig) ->
 		if userConfig.registered_at?
 			return userConfig
 
@@ -115,18 +120,22 @@ bootstrap = ->
 				console.log('Exchanging key failed, having to reregister')
 				generateRegistration(true)
 		.then ({ id }) ->
-			userConfig.registered_at = Date.now()
+			toUpdate = {}
-			userConfig.deviceId = id
+			toDelete = []
+			if !userConfig.registered_at?
+				toUpdate.registered_at = Date.now()
+			toUpdate.deviceId = id
 			osRelease.getOSVersion(config.hostOSVersionPath)
-		.then (osVersion) ->
+			.then (osVersion) ->
-			# Delete the provisioning key now, only if the OS supports it
+				# Delete the provisioning key now, only if the OS supports it
-			hasSupport = hasDeviceApiKeySupport(osVersion)
+				hasSupport = hasDeviceApiKeySupport(osVersion)
-			if hasSupport
+				if hasSupport
-				delete userConfig.apiKey
+					toDelete.push('apiKey')
-			else
+				else
-				userConfig.apiKey = userConfig.deviceApiKey
+					toUpdate.apiKey = userConfig.deviceApiKey
-			writeAndSyncFile(configPath, JSON.stringify(userConfig))
+				configJson.set(toUpdate, toDelete)
-		.return(userConfig)
+		.then ->
+			configJson.getAll()
 	.then (userConfig) ->
 		console.log('Finishing bootstrapping')
 		knex('config').whereIn('key', ['uuid', 'apiKey', 'username', 'userId', 'version']).delete()
@@ -144,20 +153,18 @@ bootstrap = ->
 		.tap ->
 			bootstrapper.doneBootstrapping()
 
-readConfig = ->
-	fs.readFileAsync(configPath, 'utf8')
-	.then(JSON.parse)
-
 generateRegistration = (forceReregister = false) ->
 	Promise.try ->
 		if forceReregister
-			userConfig.uuid = deviceRegister.generateUniqueKey()
+			configJson.set({ uuid: deviceRegister.generateUniqueKey(), deviceApiKey: deviceRegister.generateUniqueKey() })
-			userConfig.deviceApiKey = deviceRegister.generateUniqueKey()
 		else
-			userConfig.uuid ?= deviceRegister.generateUniqueKey()
+			configJson.getAll()
-			userConfig.deviceApiKey ?= deviceRegister.generateUniqueKey()
+			.then ({ uuid, deviceApiKey }) ->
-		writeAndSyncFile(configPath, JSON.stringify(userConfig))
+				uuid ?= deviceRegister.generateUniqueKey()
-		.return(userConfig.uuid)
+				deviceApiKey ?= deviceRegister.generateUniqueKey()
+				configJson.set({ uuid, deviceApiKey })
+	.then ->
+		configJson.get('uuid')
 	.catch (err) ->
 		console.log('Error generating and saving UUID: ', err)
 		Promise.delay(config.bootstrapRetryDelay)
@@ -186,23 +193,27 @@ exchangeKeyAndUpdateConfig = ->
 	# Otherwise VPN and other host services that use an API key will break.
 	#
 	# In other cases, we make the apiKey equal the deviceApiKey instead.
-	osRelease.getOSVersion(config.hostOSVersionPath)
+	Promise.join(
-	.then (osVersion) ->
+		configJson.getAll()
-		hasSupport = hasDeviceApiKeySupport(osVersion)
+		osRelease.getOSVersion(config.hostOSVersionPath)
-		if hasSupport or userConfig.apiKey != userConfig.deviceApiKey
+		(userConfig, osVersion) ->
-			console.log('Attempting key exchange')
+			hasSupport = hasDeviceApiKeySupport(osVersion)
-			exchangeKey()
+			if hasSupport or userConfig.apiKey != userConfig.deviceApiKey
-			.then ->
+				console.log('Attempting key exchange')
-				console.log('Key exchange succeeded, starting to use deviceApiKey')
+				exchangeKey()
-				if hasSupport
+				.then ->
-					delete userConfig.apiKey
+					configJson.get('deviceApiKey')
-				else
+				.then (deviceApiKey) ->
-					userConfig.apiKey = userConfig.deviceApiKey
+					console.log('Key exchange succeeded, starting to use deviceApiKey')
-				utils.setConfig('deviceApiKey', userConfig.deviceApiKey)
+					utils.setConfig('deviceApiKey', deviceApiKey)
-			.then ->
+					.then ->
-				utils.setConfig('apiKey', userConfig.deviceApiKey)
+						utils.setConfig('apiKey', deviceApiKey)
-			.then ->
+					.then ->
-				writeAndSyncFile(configPath, JSON.stringify(userConfig))
+						if hasSupport
+							configJson.set({}, [ 'apiKey' ])
+						else
+							configJson.set(apiKey: deviceApiKey)
+	)
 
 exchangeKeyOrRetry = do ->
 	_failedExchanges = 0
@@ -217,31 +228,34 @@ exchangeKeyOrRetry = do ->
 
 bootstrapper.done = new Promise (resolve) ->
 	bootstrapper.doneBootstrapping = ->
-		bootstrapper.bootstrapped = true
+		configJson.getAll()
-		resolve(userConfig)
+		.then (userConfig) ->
-		# If we're still using an old api key we can try to exchange it for a valid device key
+			bootstrapper.bootstrapped = true
-		# This will only be the case when the supervisor/OS has been updated.
+			resolve(userConfig)
-		if userConfig.apiKey?
+			# If we're still using an old api key we can try to exchange it for a valid device key
-			exchangeKeyOrRetry()
+			# This will only be the case when the supervisor/OS has been updated.
-		else
+			if userConfig.apiKey?
-			Promise.join(
+				exchangeKeyOrRetry()
-				knex('config').select('value').where(key: 'apiKey')
+			else
-				knex('config').select('value').where(key: 'deviceApiKey')
+				Promise.join(
-				([ apiKey ], [ deviceApiKey ]) ->
+					knex('config').select('value').where(key: 'apiKey')
-					if !deviceApiKey?.value
+					knex('config').select('value').where(key: 'deviceApiKey')
-						# apiKey in the DB is actually the deviceApiKey, but it was
+					([ apiKey ], [ deviceApiKey ]) ->
-						# exchanged in a supervisor version that didn't save it to the DB
+						if !deviceApiKey?.value
-						# (which mainly affects the RESIN_API_KEY env var)
+							# apiKey in the DB is actually the deviceApiKey, but it was
-						knex('config').insert({ key: 'deviceApiKey', value: apiKey.value })
+							# exchanged in a supervisor version that didn't save it to the DB
-			)
+							# (which mainly affects the RESIN_API_KEY env var)
+							knex('config').insert({ key: 'deviceApiKey', value: apiKey.value })
+				)
 		return
 
 bootstrapper.bootstrapped = false
 bootstrapper.startBootstrapping = ->
 	# Load config file
-	readConfig()
+	configJson.init()
-	.then (configFromFile) ->
+	.then ->
-		userConfig = configFromFile
+		configJson.getAll()
+	.then (userConfig) ->
 		bootstrapper.offlineMode = !Boolean(config.apiEndpoint) or Boolean(userConfig.supervisorOfflineMode)
 		knex('config').select('value').where(key: 'uuid')
 	.then ([ uuid ]) ->

src/config-json.coffee

@@ -0,0 +1,46 @@
+Promise = require 'bluebird'
+_ = require 'lodash'
+configPath = '/boot/config.json'
+Lock = require 'rwlock'
+fs = Promise.promisifyAll(require('fs'))
+{ writeAndSyncFile } = require './lib/fs-utils'
+
+lock = new Lock()
+writeLock = Promise.promisify(lock.async.writeLock)
+readLock = Promise.promisify(lock.async.readLock)
+withWriteLock = do ->
+	_takeLock = ->
+		writeLock('config')
+		.disposer (release) ->
+			release()
+	return (func) ->
+		Promise.using(_takeLock(), func)
+withReadLock = do ->
+	_takeLock = ->
+		readLock('config')
+		.disposer (release) ->
+			release()
+	return (func) ->
+		Promise.using(_takeLock(), func)
+
+# write-through cache of the config.json file
+userConfig = null
+exports.init = ->
+	fs.readFileAsync(configPath)
+	.then (conf) ->
+		userConfig = JSON.parse(conf)
+
+exports.get = (key) ->
+	withReadLock ->
+		return userConfig[key]
+
+exports.getAll = ->
+	withReadLock ->
+		return _.clone(userConfig)
+
+exports.set = (vals = {}, keysToDelete = []) ->
+	withWriteLock ->
+		_.merge(userConfig, vals)
+		for key in keysToDelete
+			delete userConfig[key]
+		writeAndSyncFile(configPath, JSON.stringify(userConfig))

src/host-config.coffee

@@ -0,0 +1,128 @@
+Promise = require 'bluebird'
+_ = require 'lodash'
+utils = require './utils'
+path = require 'path'
+config = require './config'
+fs = Promise.promisifyAll(require('fs'))
+configJson = require './config-json'
+{ writeFileAtomic } = require './lib/fs-utils'
+mkdirp = Promise.promisify(require('mkdirp'))
+
+ENOENT = (err) -> err.code is 'ENOENT'
+
+redsocksHeader = '''
+	base {
+		log_debug = off;
+		log_info = on;
+		log = stderr;
+		daemon = off;
+		redirector = iptables;
+	}
+
+	redsocks {
+		local_ip = 127.0.0.1;
+		local_port = 12345;
+
+	'''
+
+redsocksFooter = '}\n'
+
+proxyFields = [ 'type', 'ip', 'port', 'login', 'password' ]
+
+proxyBasePath = path.join('/mnt/root', config.bootMountPoint, 'system-proxy')
+redsocksConfPath = path.join(proxyBasePath, 'redsocks.conf')
+noProxyPath = path.join(proxyBasePath, 'no_proxy')
+
+readProxy = ->
+	fs.readFileAsync(redsocksConfPath)
+	.then (redsocksConf) ->
+		lines = new String(redsocksConf).split('\n')
+		conf = {}
+		for line in lines
+			for proxyField in proxyFields
+				if proxyField in [ 'login', 'password' ]
+					m = line.match(new RegExp(proxyField + '\\s*=\\s*\"(.*)\"\\s*;'))
+				else
+					m = line.match(new RegExp(proxyField + '\\s*=\\s*([^;\\s]*)\\s*;'))
+				if m?
+					conf[proxyField] = m[1]
+		return conf
+	.catch ENOENT, ->
+		return null
+	.then (conf) ->
+		if !conf?
+			return null
+		else
+			fs.readFileAsync(noProxyPath)
+			.then (noProxy) ->
+				conf.noProxy = new String(noProxy).split('\n')
+				return conf
+			.catch ENOENT, ->
+				return conf
+
+generateRedsocksConfEntries = (conf) ->
+	val = ''
+	for field in proxyFields
+		if conf[field]?
+			v = conf[field]
+			if field in [ 'login', 'password' ]
+				v = "\"#{v}\""
+			val += "\t#{field} = #{v};\n"
+	return val
+
+setProxy = (conf) ->
+	Promise.try ->
+		if _.isEmpty(conf)
+			fs.unlinkAsync(redsocksConfPath)
+			.catch(ENOENT, _.noop)
+			.then ->
+				fs.unlinkAsync(noProxyPath)
+			.catch(ENOENT, _.noop)
+		else
+			mkdirp(proxyBasePath)
+			.then ->
+				if _.isArray(conf.noProxy)
+					writeFileAtomic(noProxyPath, conf.noProxy.join('\n'))
+			.then ->
+				redsocksConf = ''
+				redsocksConf += redsocksHeader
+				redsocksConf += generateRedsocksConfEntries(conf)
+				redsocksConf += redsocksFooter
+				writeFileAtomic(redsocksConfPath, redsocksConf)
+	.then ->
+		utils.restartSystemdService('resin-proxy-config')
+	.then ->
+		utils.restartSystemdService('redsocks')
+
+hostnamePath = '/mnt/root/etc/hostname'
+readHostname = ->
+	fs.readFileAsync(hostnamePath)
+	.then (hostnameData) ->
+		return _.trim(new String(hostnameData))
+
+setHostname = (val) ->
+	configJson.set(hostname: val)
+	.then ->
+		utils.restartSystemdService('resin-hostname')
+
+
+exports.get = ->
+	Promise.join(
+		readProxy()
+		readHostname()
+		(proxy, hostname) ->
+			return {
+				network: {
+					proxy
+					hostname
+				}
+			}
+	)
+
+exports.patch = (conf) ->
+	Promise.try ->
+		if !_.isUndefined(conf?.network?.proxy)
+			setProxy(conf.network.proxy)
+	.then ->
+		if !_.isUndefined(conf?.network?.hostname)
+			setHostname(conf.network.hostname)

src/lib/fs-utils.coffee

@@ -0,0 +1,16 @@
+Promise = require 'bluebird'
+fs = Promise.promisifyAll(require('fs'))
+
+exports.writeAndSyncFile = (path, data) ->
+	fs.openAsync(path, 'w')
+	.then (fd) ->
+		fs.writeAsync(fd, data, 0, 'utf8')
+		.then ->
+			fs.fsyncAsync(fd)
+		.then ->
+			fs.closeAsync(fd)
+
+exports.writeFileAtomic = (path, data) ->
+	exports.writeAndSyncFile("#{path}.new", data)
+	.then ->
+		fs.renameAsync("#{path}.new", path)

src/utils.coffee

@@ -236,6 +236,14 @@ exports.vpnControl = (val, logMessage, { initial = false } = {}) ->
 			return false
 	.catchReturn(false)
 
+exports.restartSystemdService = (serviceName) ->
+	gosuper.postAsync('/v1/restart-service', { json: true, body: Name: serviceName })
+	.spread (response, body) ->
+		if response.statusCode != 200
+			err = new Error("Error restarting service #{serviceName}: #{response.statusCode} #{body}")
+			err.statusCode = response.statusCode
+			throw err
+
 exports.AppNotFoundError = class AppNotFoundError extends TypedError
 
 exports.getKnexApp = (appId, columns) ->