From cff789ebfad31e0caa88fc81bc2ba2a3b00f5bdc Mon Sep 17 00:00:00 2001 From: Pablo Carranza Velez Date: Mon, 15 Jan 2018 18:50:03 -0300 Subject: [PATCH] Implement an API for proxy and hostname configuration, and centralize management of config.json Change-Type: minor Signed-off-by: Pablo Carranza Velez --- gosuper/gosuper/api.go | 30 ++++++ gosuper/gosuper/main.go | 1 + package.json | 3 +- src/api.coffee | 16 ++- src/bootstrap.coffee | 230 +++++++++++++++++++++------------------- src/config-json.coffee | 46 ++++++++ src/host-config.coffee | 128 ++++++++++++++++++++++ src/lib/fs-utils.coffee | 16 +++ src/utils.coffee | 8 ++ 9 files changed, 368 insertions(+), 110 deletions(-) create mode 100644 src/config-json.coffee create mode 100644 src/host-config.coffee create mode 100644 src/lib/fs-utils.coffee diff --git a/gosuper/gosuper/api.go b/gosuper/gosuper/api.go index 0527dc8a..abd3fa41 100644 --- a/gosuper/gosuper/api.go +++ b/gosuper/gosuper/api.go @@ -38,6 +38,10 @@ type LogToDisplayBody struct { Enable bool } +type RestartServiceBody struct { + Name string +} + func jsonResponse(writer http.ResponseWriter, response interface{}, status int) { jsonBody, err := json.Marshal(response) if err != nil { @@ -278,3 +282,29 @@ func LogToDisplayControl(writer http.ResponseWriter, request *http.Request) { sendResponse(true, "", http.StatusOK) } } + +//RestartService is used to restart a systemd service by name +func RestartService(writer http.ResponseWriter, request *http.Request) { + sendResponse, sendError := responseSenders(writer) + var body RestartServiceBody + if err := parseJSONBody(&body, request); err != nil { + log.Println(err) + sendResponse("Error", err.Error(), http.StatusBadRequest) + return + } + + if systemd.Dbus == nil { + sendError(fmt.Errorf("Systemd dbus unavailable, cannot restart service.")) + return + } + + actionDescr := "Service Restart" + + if _, err := systemd.Dbus.RestartUnit(body.Name+".service", "fail", nil); err != nil { + log.Printf("%s: %s\n", actionDescr, err) + sendError(err) + return + } + log.Printf("%sd\n", actionDescr) + sendResponse("OK", "", http.StatusOK) +} diff --git a/gosuper/gosuper/main.go b/gosuper/gosuper/main.go index e27adc98..579fe2da 100644 --- a/gosuper/gosuper/main.go +++ b/gosuper/gosuper/main.go @@ -26,6 +26,7 @@ func setupApi(router *mux.Router) { apiv1.HandleFunc("/shutdown", ShutdownHandler).Methods("POST") apiv1.HandleFunc("/vpncontrol", VPNControl).Methods("POST") apiv1.HandleFunc("/set-log-to-display", LogToDisplayControl).Methods("POST") + apiv1.HandleFunc("/restart-service", RestartService).Methods("POST") } func startApi(listenAddress string, router *mux.Router) { diff --git a/package.json b/package.json index 67c9db39..06a5fab2 100644 --- a/package.json +++ b/package.json @@ -40,6 +40,7 @@ "log-timestamp": "^0.1.2", "memoizee": "^0.4.1", "mixpanel": "0.0.20", + "mkdirp": "^0.5.1", "network-checker": "~0.0.5", "node-loader": "^0.6.0", "null-loader": "^0.1.1", @@ -59,4 +60,4 @@ "versionist": "^2.8.0", "webpack": "^3.0.0" } -} \ No newline at end of file +} diff --git a/src/api.coffee b/src/api.coffee index 7810304e..7a25ad66 100644 --- a/src/api.coffee +++ b/src/api.coffee @@ -7,6 +7,7 @@ config = require './config' device = require './device' _ = require 'lodash' proxyvisor = require './proxyvisor' +hostConfig = require './host-config' module.exports = (application) -> authenticate = (req, res, next) -> @@ -208,7 +209,6 @@ module.exports = (application) -> .catch (err) -> res.status(503).send(err?.message or err or 'Unknown error') - # Expires the supervisor's API key and generates a new one. # It also communicates the new key to the Resin API. unparsedRouter.post '/v1/regenerate-api-key', (req, res) -> @@ -219,6 +219,20 @@ module.exports = (application) -> .catch (err) -> res.status(503).send(err?.message or err or 'Unknown error') + parsedRouter.get '/v1/device/host-config', (req, res) -> + hostConfig.get() + .then (conf) -> + res.json(conf) + .catch (err) -> + res.status(503).send(err?.message or err or 'Unknown error') + + parsedRouter.patch '/v1/device/host-config', (req, res) -> + hostConfig.patch(req.body) + .then -> + res.status(200).send('OK') + .catch (err) -> + res.status(503).send(err?.message or err or 'Unknown error') + unparsedRouter.get '/v1/device', (req, res) -> res.json(device.getState()) diff --git a/src/bootstrap.coffee b/src/bootstrap.coffee index c2943866..14e09f38 100644 --- a/src/bootstrap.coffee +++ b/src/bootstrap.coffee @@ -5,7 +5,6 @@ deviceRegister = require 'resin-register-device' { resinApi, request } = require './request' fs = Promise.promisifyAll(require('fs')) config = require './config' -configPath = '/boot/config.json' appsPath = '/boot/apps.json' _ = require 'lodash' deviceConfig = require './device-config' @@ -13,23 +12,13 @@ TypedError = require 'typed-error' osRelease = require './lib/os-release' semver = require 'semver' semverRegex = require('semver-regex') - -userConfig = {} +configJson = require './config-json' DuplicateUuidError = (err) -> _.startsWith(err.message, '"uuid" must be unique') exports.ExchangeKeyError = class ExchangeKeyError extends TypedError bootstrapper = {} -writeAndSyncFile = (path, data) -> - fs.openAsync(path, 'w') - .then (fd) -> - fs.writeAsync(fd, data, 0, 'utf8') - .then -> - fs.fsyncAsync(fd) - .then -> - fs.closeAsync(fd) - loadPreloadedApps = -> devConfig = {} knex('app').select() @@ -37,27 +26,29 @@ loadPreloadedApps = -> if apps.length > 0 console.log('Preloaded apps already loaded, skipping') return - fs.readFileAsync(appsPath, 'utf8') - .then(JSON.parse) - .map (app) -> - utils.extendEnvVars(app.env, userConfig.uuid, userConfig.deviceApiKey, app.appId, app.name, app.commit) - .then (extendedEnv) -> - app.env = JSON.stringify(extendedEnv) - app.markedForDeletion = false - _.merge(devConfig, app.config) - app.config = JSON.stringify(app.config) - knex('app').insert(app) - .then -> - deviceConfig.set({ targetValues: devConfig }) + configJson.getAll() + .then (userConfig) -> + fs.readFileAsync(appsPath, 'utf8') + .then(JSON.parse) + .map (app) -> + utils.extendEnvVars(app.env, userConfig.uuid, userConfig.deviceApiKey, app.appId, app.name, app.commit) + .then (extendedEnv) -> + app.env = JSON.stringify(extendedEnv) + app.markedForDeletion = false + _.merge(devConfig, app.config) + app.config = JSON.stringify(app.config) + knex('app').insert(app) + .then -> + deviceConfig.set({ targetValues: devConfig }) .catch (err) -> utils.mixpanelTrack('Loading preloaded apps failed', { error: err }) -fetchDevice = (apiKey) -> +fetchDevice = (uuid, apiKey) -> resinApi.get resource: 'device' options: filter: - uuid: userConfig.uuid + uuid: uuid customOptions: apikey: apiKey .get(0) @@ -65,33 +56,47 @@ fetchDevice = (apiKey) -> .timeout(config.apiTimeout) exchangeKey = -> - Promise.try -> - # If we have an existing device key we first check if it's valid, because if it is we can just use that - if userConfig.deviceApiKey? - fetchDevice(userConfig.deviceApiKey) - .then (device) -> - if device? - return device - # If it's not valid/doesn't exist then we try to use the user/provisioning api key for the exchange - fetchDevice(userConfig.apiKey) + configJson.getAll() + .then (userConfig) -> + Promise.try -> + # If we have an existing device key we first check if it's valid, because if it is we can just use that + if userConfig.deviceApiKey? + fetchDevice(userConfig.uuid, userConfig.deviceApiKey) .then (device) -> - if not device? - throw new ExchangeKeyError("Couldn't fetch device with provisioning key") - # We found the device, we can try to register a working device key for it - userConfig.deviceApiKey ?= deviceRegister.generateUniqueKey() - request.postAsync("#{config.apiEndpoint}/api-key/device/#{device.id}/device-key?apikey=#{userConfig.apiKey}", { - json: true - body: - apiKey: userConfig.deviceApiKey - }) - .spread (res, body) -> - if res.statusCode != 200 - throw new ExchangeKeyError("Couldn't register device key with provisioning key") - .return(device) + if device? + return device + # If it's not valid/doesn't exist then we try to use the user/provisioning api key for the exchange + fetchDevice(userConfig.uuid, userConfig.apiKey) + .then (device) -> + if not device? + throw new ExchangeKeyError("Couldn't fetch device with provisioning key") + # We found the device, we can try to register a working device key for it + Promise.try -> + if !userConfig.deviceApiKey? + deviceApiKey = deviceRegister.generateUniqueKey() + configJson.set({ deviceApiKey }) + .return(deviceApiKey) + else + return userConfig.deviceApiKey + .then (deviceApiKey) -> + request.postAsync("#{config.apiEndpoint}/api-key/device/#{device.id}/device-key?apikey=#{userConfig.apiKey}", { + json: true + body: + apiKey: deviceApiKey + }) + .spread (res, body) -> + if res.statusCode != 200 + throw new ExchangeKeyError("Couldn't register device key with provisioning key") + .return(device) bootstrap = -> - Promise.try -> - userConfig.deviceType ?= 'raspberry-pi' + configJson.get('deviceType') + .then (deviceType) -> + if !deviceType? + configJson.set(deviceType: 'raspberry-pi') + .then -> + configJson.getAll() + .then (userConfig) -> if userConfig.registered_at? return userConfig @@ -115,18 +120,22 @@ bootstrap = -> console.log('Exchanging key failed, having to reregister') generateRegistration(true) .then ({ id }) -> - userConfig.registered_at = Date.now() - userConfig.deviceId = id + toUpdate = {} + toDelete = [] + if !userConfig.registered_at? + toUpdate.registered_at = Date.now() + toUpdate.deviceId = id osRelease.getOSVersion(config.hostOSVersionPath) - .then (osVersion) -> - # Delete the provisioning key now, only if the OS supports it - hasSupport = hasDeviceApiKeySupport(osVersion) - if hasSupport - delete userConfig.apiKey - else - userConfig.apiKey = userConfig.deviceApiKey - writeAndSyncFile(configPath, JSON.stringify(userConfig)) - .return(userConfig) + .then (osVersion) -> + # Delete the provisioning key now, only if the OS supports it + hasSupport = hasDeviceApiKeySupport(osVersion) + if hasSupport + toDelete.push('apiKey') + else + toUpdate.apiKey = userConfig.deviceApiKey + configJson.set(toUpdate, toDelete) + .then -> + configJson.getAll() .then (userConfig) -> console.log('Finishing bootstrapping') knex('config').whereIn('key', ['uuid', 'apiKey', 'username', 'userId', 'version']).delete() @@ -144,20 +153,18 @@ bootstrap = -> .tap -> bootstrapper.doneBootstrapping() -readConfig = -> - fs.readFileAsync(configPath, 'utf8') - .then(JSON.parse) - generateRegistration = (forceReregister = false) -> Promise.try -> if forceReregister - userConfig.uuid = deviceRegister.generateUniqueKey() - userConfig.deviceApiKey = deviceRegister.generateUniqueKey() + configJson.set({ uuid: deviceRegister.generateUniqueKey(), deviceApiKey: deviceRegister.generateUniqueKey() }) else - userConfig.uuid ?= deviceRegister.generateUniqueKey() - userConfig.deviceApiKey ?= deviceRegister.generateUniqueKey() - writeAndSyncFile(configPath, JSON.stringify(userConfig)) - .return(userConfig.uuid) + configJson.getAll() + .then ({ uuid, deviceApiKey }) -> + uuid ?= deviceRegister.generateUniqueKey() + deviceApiKey ?= deviceRegister.generateUniqueKey() + configJson.set({ uuid, deviceApiKey }) + .then -> + configJson.get('uuid') .catch (err) -> console.log('Error generating and saving UUID: ', err) Promise.delay(config.bootstrapRetryDelay) @@ -186,23 +193,27 @@ exchangeKeyAndUpdateConfig = -> # Otherwise VPN and other host services that use an API key will break. # # In other cases, we make the apiKey equal the deviceApiKey instead. - osRelease.getOSVersion(config.hostOSVersionPath) - .then (osVersion) -> - hasSupport = hasDeviceApiKeySupport(osVersion) - if hasSupport or userConfig.apiKey != userConfig.deviceApiKey - console.log('Attempting key exchange') - exchangeKey() - .then -> - console.log('Key exchange succeeded, starting to use deviceApiKey') - if hasSupport - delete userConfig.apiKey - else - userConfig.apiKey = userConfig.deviceApiKey - utils.setConfig('deviceApiKey', userConfig.deviceApiKey) - .then -> - utils.setConfig('apiKey', userConfig.deviceApiKey) - .then -> - writeAndSyncFile(configPath, JSON.stringify(userConfig)) + Promise.join( + configJson.getAll() + osRelease.getOSVersion(config.hostOSVersionPath) + (userConfig, osVersion) -> + hasSupport = hasDeviceApiKeySupport(osVersion) + if hasSupport or userConfig.apiKey != userConfig.deviceApiKey + console.log('Attempting key exchange') + exchangeKey() + .then -> + configJson.get('deviceApiKey') + .then (deviceApiKey) -> + console.log('Key exchange succeeded, starting to use deviceApiKey') + utils.setConfig('deviceApiKey', deviceApiKey) + .then -> + utils.setConfig('apiKey', deviceApiKey) + .then -> + if hasSupport + configJson.set({}, [ 'apiKey' ]) + else + configJson.set(apiKey: deviceApiKey) + ) exchangeKeyOrRetry = do -> _failedExchanges = 0 @@ -217,31 +228,34 @@ exchangeKeyOrRetry = do -> bootstrapper.done = new Promise (resolve) -> bootstrapper.doneBootstrapping = -> - bootstrapper.bootstrapped = true - resolve(userConfig) - # If we're still using an old api key we can try to exchange it for a valid device key - # This will only be the case when the supervisor/OS has been updated. - if userConfig.apiKey? - exchangeKeyOrRetry() - else - Promise.join( - knex('config').select('value').where(key: 'apiKey') - knex('config').select('value').where(key: 'deviceApiKey') - ([ apiKey ], [ deviceApiKey ]) -> - if !deviceApiKey?.value - # apiKey in the DB is actually the deviceApiKey, but it was - # exchanged in a supervisor version that didn't save it to the DB - # (which mainly affects the RESIN_API_KEY env var) - knex('config').insert({ key: 'deviceApiKey', value: apiKey.value }) - ) + configJson.getAll() + .then (userConfig) -> + bootstrapper.bootstrapped = true + resolve(userConfig) + # If we're still using an old api key we can try to exchange it for a valid device key + # This will only be the case when the supervisor/OS has been updated. + if userConfig.apiKey? + exchangeKeyOrRetry() + else + Promise.join( + knex('config').select('value').where(key: 'apiKey') + knex('config').select('value').where(key: 'deviceApiKey') + ([ apiKey ], [ deviceApiKey ]) -> + if !deviceApiKey?.value + # apiKey in the DB is actually the deviceApiKey, but it was + # exchanged in a supervisor version that didn't save it to the DB + # (which mainly affects the RESIN_API_KEY env var) + knex('config').insert({ key: 'deviceApiKey', value: apiKey.value }) + ) return bootstrapper.bootstrapped = false bootstrapper.startBootstrapping = -> # Load config file - readConfig() - .then (configFromFile) -> - userConfig = configFromFile + configJson.init() + .then -> + configJson.getAll() + .then (userConfig) -> bootstrapper.offlineMode = !Boolean(config.apiEndpoint) or Boolean(userConfig.supervisorOfflineMode) knex('config').select('value').where(key: 'uuid') .then ([ uuid ]) -> diff --git a/src/config-json.coffee b/src/config-json.coffee new file mode 100644 index 00000000..b1c878b1 --- /dev/null +++ b/src/config-json.coffee @@ -0,0 +1,46 @@ +Promise = require 'bluebird' +_ = require 'lodash' +configPath = '/boot/config.json' +Lock = require 'rwlock' +fs = Promise.promisifyAll(require('fs')) +{ writeAndSyncFile } = require './lib/fs-utils' + +lock = new Lock() +writeLock = Promise.promisify(lock.async.writeLock) +readLock = Promise.promisify(lock.async.readLock) +withWriteLock = do -> + _takeLock = -> + writeLock('config') + .disposer (release) -> + release() + return (func) -> + Promise.using(_takeLock(), func) +withReadLock = do -> + _takeLock = -> + readLock('config') + .disposer (release) -> + release() + return (func) -> + Promise.using(_takeLock(), func) + +# write-through cache of the config.json file +userConfig = null +exports.init = -> + fs.readFileAsync(configPath) + .then (conf) -> + userConfig = JSON.parse(conf) + +exports.get = (key) -> + withReadLock -> + return userConfig[key] + +exports.getAll = -> + withReadLock -> + return _.clone(userConfig) + +exports.set = (vals = {}, keysToDelete = []) -> + withWriteLock -> + _.merge(userConfig, vals) + for key in keysToDelete + delete userConfig[key] + writeAndSyncFile(configPath, JSON.stringify(userConfig)) diff --git a/src/host-config.coffee b/src/host-config.coffee new file mode 100644 index 00000000..e6a3d083 --- /dev/null +++ b/src/host-config.coffee @@ -0,0 +1,128 @@ +Promise = require 'bluebird' +_ = require 'lodash' +utils = require './utils' +path = require 'path' +config = require './config' +fs = Promise.promisifyAll(require('fs')) +configJson = require './config-json' +{ writeFileAtomic } = require './lib/fs-utils' +mkdirp = Promise.promisify(require('mkdirp')) + +ENOENT = (err) -> err.code is 'ENOENT' + +redsocksHeader = ''' + base { + log_debug = off; + log_info = on; + log = stderr; + daemon = off; + redirector = iptables; + } + + redsocks { + local_ip = 127.0.0.1; + local_port = 12345; + + ''' + +redsocksFooter = '}\n' + +proxyFields = [ 'type', 'ip', 'port', 'login', 'password' ] + +proxyBasePath = path.join('/mnt/root', config.bootMountPoint, 'system-proxy') +redsocksConfPath = path.join(proxyBasePath, 'redsocks.conf') +noProxyPath = path.join(proxyBasePath, 'no_proxy') + +readProxy = -> + fs.readFileAsync(redsocksConfPath) + .then (redsocksConf) -> + lines = new String(redsocksConf).split('\n') + conf = {} + for line in lines + for proxyField in proxyFields + if proxyField in [ 'login', 'password' ] + m = line.match(new RegExp(proxyField + '\\s*=\\s*\"(.*)\"\\s*;')) + else + m = line.match(new RegExp(proxyField + '\\s*=\\s*([^;\\s]*)\\s*;')) + if m? + conf[proxyField] = m[1] + return conf + .catch ENOENT, -> + return null + .then (conf) -> + if !conf? + return null + else + fs.readFileAsync(noProxyPath) + .then (noProxy) -> + conf.noProxy = new String(noProxy).split('\n') + return conf + .catch ENOENT, -> + return conf + +generateRedsocksConfEntries = (conf) -> + val = '' + for field in proxyFields + if conf[field]? + v = conf[field] + if field in [ 'login', 'password' ] + v = "\"#{v}\"" + val += "\t#{field} = #{v};\n" + return val + +setProxy = (conf) -> + Promise.try -> + if _.isEmpty(conf) + fs.unlinkAsync(redsocksConfPath) + .catch(ENOENT, _.noop) + .then -> + fs.unlinkAsync(noProxyPath) + .catch(ENOENT, _.noop) + else + mkdirp(proxyBasePath) + .then -> + if _.isArray(conf.noProxy) + writeFileAtomic(noProxyPath, conf.noProxy.join('\n')) + .then -> + redsocksConf = '' + redsocksConf += redsocksHeader + redsocksConf += generateRedsocksConfEntries(conf) + redsocksConf += redsocksFooter + writeFileAtomic(redsocksConfPath, redsocksConf) + .then -> + utils.restartSystemdService('resin-proxy-config') + .then -> + utils.restartSystemdService('redsocks') + +hostnamePath = '/mnt/root/etc/hostname' +readHostname = -> + fs.readFileAsync(hostnamePath) + .then (hostnameData) -> + return _.trim(new String(hostnameData)) + +setHostname = (val) -> + configJson.set(hostname: val) + .then -> + utils.restartSystemdService('resin-hostname') + + +exports.get = -> + Promise.join( + readProxy() + readHostname() + (proxy, hostname) -> + return { + network: { + proxy + hostname + } + } + ) + +exports.patch = (conf) -> + Promise.try -> + if !_.isUndefined(conf?.network?.proxy) + setProxy(conf.network.proxy) + .then -> + if !_.isUndefined(conf?.network?.hostname) + setHostname(conf.network.hostname) diff --git a/src/lib/fs-utils.coffee b/src/lib/fs-utils.coffee new file mode 100644 index 00000000..b05e8bea --- /dev/null +++ b/src/lib/fs-utils.coffee @@ -0,0 +1,16 @@ +Promise = require 'bluebird' +fs = Promise.promisifyAll(require('fs')) + +exports.writeAndSyncFile = (path, data) -> + fs.openAsync(path, 'w') + .then (fd) -> + fs.writeAsync(fd, data, 0, 'utf8') + .then -> + fs.fsyncAsync(fd) + .then -> + fs.closeAsync(fd) + +exports.writeFileAtomic = (path, data) -> + exports.writeAndSyncFile("#{path}.new", data) + .then -> + fs.renameAsync("#{path}.new", path) diff --git a/src/utils.coffee b/src/utils.coffee index a9fbc98c..d8ead9dd 100644 --- a/src/utils.coffee +++ b/src/utils.coffee @@ -236,6 +236,14 @@ exports.vpnControl = (val, logMessage, { initial = false } = {}) -> return false .catchReturn(false) +exports.restartSystemdService = (serviceName) -> + gosuper.postAsync('/v1/restart-service', { json: true, body: Name: serviceName }) + .spread (response, body) -> + if response.statusCode != 200 + err = new Error("Error restarting service #{serviceName}: #{response.statusCode} #{body}") + err.statusCode = response.statusCode + throw err + exports.AppNotFoundError = class AppNotFoundError extends TypedError exports.getKnexApp = (appId, columns) ->