Merge pull request #238 from resin-io/235-dependent-apps

Dependent apps and devices

CHANGELOG.md

@@ -1,3 +1,10 @@
+* Switch to v2 api to be able to set is_online [Page]
+* Implement proxyvisor API with dependent device handling [Pablo]
+* Use the state endpoint from the API to get the full device state [Pablo]
+* Add a deviceConfig db table to store host config separately, and allow deleting config.txt entries [Pablo]
+* Expose RESIN_APP_NAME, RESIN_APP_RELEASE, RESIN_DEVICE_NAME_AT_INIT, RESIN_DEVICE_TYPE and RESIN_HOST_OS_VERSION env vars [Pablo]
+* Add missing error handler on a stream in docker-utils [Pablo]
+
 # v2.4.0
 
 * On cleanup, force removal for images and containers, and remove container volumes [Pablo]

src/api.coffee

@@ -3,12 +3,12 @@ utils = require './utils'
 express = require 'express'
 bodyParser = require 'body-parser'
 bufferEq = require 'buffer-equal-constant-time'
-request = require 'request'
 config = require './config'
 device = require './device'
 dockerUtils = require './docker-utils'
 _ = require 'lodash'
 compose = require './compose'
+proxyvisor = require './proxyvisor'
 
 module.exports = (application) ->
 	api = express()
@@ -235,5 +235,6 @@ module.exports = (application) ->
 
 	api.use(unparsedRouter)
 	api.use(parsedRouter)
+	api.use(proxyvisor.router)
 
 	return api

src/app.coffee

@@ -7,7 +7,6 @@ knex = require './db'
 utils = require './utils'
 bootstrap = require './bootstrap'
 config = require './config'
-request = require 'request'
 _ = require 'lodash'
 
 knex.init.then ->

src/application.coffee

@@ -2,7 +2,6 @@ _ = require 'lodash'
 url = require 'url'
 Lock = require 'rwlock'
 knex = require './db'
-path = require 'path'
 config = require './config'
 dockerUtils = require './docker-utils'
 Promise = require 'bluebird'
@@ -15,6 +14,7 @@ bootstrap = require './bootstrap'
 TypedError = require 'typed-error'
 fs = Promise.promisifyAll(require('fs'))
 JSONStream = require 'JSONStream'
+proxyvisor = require './proxyvisor'
 
 class UpdatesLockedError extends TypedError
 ImageNotFoundError = (err) ->
@@ -166,7 +166,7 @@ isValidPort = (port) ->
 	maybePort = parseInt(port, 10)
 	return parseFloat(port) is maybePort and maybePort > 0 and maybePort < 65535
 
-fetch = (app) ->
+fetch = (app, setDeviceUpdateState = true) ->
 	onProgress = (progress) ->
 		device.updateState(download_progress: progress.percentage)
 
@@ -176,16 +176,16 @@ fetch = (app) ->
 		device.updateState(status: 'Downloading', download_progress: 0)
 
 		Promise.try ->
-			JSON.parse(app.env)
+			conf = JSON.parse(app.config)
-		.then (env) ->
+
-			if env['RESIN_SUPERVISOR_DELTA'] == '1'
+			if conf['RESIN_SUPERVISOR_DELTA'] == '1'
 				dockerUtils.rsyncImageWithProgress(app.imageId, onProgress)
 			else
 				dockerUtils.fetchImageWithProgress(app.imageId, onProgress)
 		.then ->
 			logSystemEvent(logTypes.downloadAppSuccess, app)
 			device.updateState(status: 'Idle', download_progress: null)
-			device.setUpdateState(update_downloaded: true)
+			device.setUpdateState(update_downloaded: true) if setDeviceUpdateState
 			docker.getImage(app.imageId).inspectAsync()
 		.catch (err) ->
 			logSystemEvent(logTypes.downloadAppError, app, err)
@@ -206,8 +206,8 @@ application.start = start = (app) ->
 	binds = utils.defaultBinds(app.appId)
 	Promise.try ->
 		# Parse the env vars before trying to access them, that's because they have to be stringified for knex..
-		JSON.parse(app.env)
+		return [ JSON.parse(app.env), JSON.parse(app.config) ]
-	.then (env) ->
+	.spread (env, conf) ->
 		if env.PORT?
 			portList = env.PORT
 			.split(',')
@@ -258,7 +258,7 @@ application.start = start = (app) ->
 			if portList?
 				portList.forEach (port) ->
 					ports[port + '/tcp'] = [ HostPort: port ]
-			restartPolicy = createRestartPolicy({ name: env['RESIN_APP_RESTART_POLICY'], maximumRetryCount: env['RESIN_APP_RESTART_RETRIES'] })
+			restartPolicy = createRestartPolicy({ name: conf['RESIN_APP_RESTART_POLICY'], maximumRetryCount: conf['RESIN_APP_RESTART_RETRIES'] })
 			shouldMountKmod(app.imageId)
 			.then (shouldMount) ->
 				binds.push('/bin/kmod:/bin/kmod:ro') if shouldMount
@@ -322,21 +322,6 @@ createRestartPolicy = ({ name, maximumRetryCount }) ->
 		policy.MaximumRetryCount = maximumRetryCount
 	return policy
 
-getEnvironment = do ->
-	envApiEndpoint = url.resolve(config.apiEndpoint, '/environment')
-
-	return (appId, deviceId, apiKey) ->
-
-		requestParams = _.extend
-			method: 'GET'
-			url: "#{envApiEndpoint}?deviceId=#{deviceId}&appId=#{appId}&apikey=#{apiKey}"
-		, cachedResinApi.passthrough
-
-		cachedResinApi._request(requestParams)
-		.catch (err) ->
-			console.error("Failed to get environment for device #{deviceId}, app #{appId}. #{err}")
-			throw err
-
 lockPath = (app) ->
 	appId = app.appId ? app
 	return "/mnt/root#{config.dataPath}/#{appId}/resin-updates.lock"
@@ -391,33 +376,39 @@ apiPollInterval = (val) ->
 	clearInterval(updateStatus.intervalHandle)
 	application.poll()
 
-specialActionEnvVars =
+specialActionConfigVars =
-	'RESIN_OVERRIDE_LOCK': null # This one is in use, so we keep backwards comp.
-	'RESIN_SUPERVISOR_DELTA': null
-	'RESIN_SUPERVISOR_UPDATE_STRATEGY': null
-	'RESIN_SUPERVISOR_HANDOVER_TIMEOUT': null
-	'RESIN_SUPERVISOR_OVERRIDE_LOCK': null
 	'RESIN_SUPERVISOR_VPN_CONTROL': utils.vpnControl
 	'RESIN_SUPERVISOR_CONNECTIVITY_CHECK': utils.enableConnectivityCheck
 	'RESIN_SUPERVISOR_POLL_INTERVAL': apiPollInterval
 	'RESIN_SUPERVISOR_LOG_CONTROL': utils.resinLogControl
 
-executedSpecialActionEnvVars = {}
+executedSpecialActionConfigVars = {}
 
-executeSpecialActionsAndHostConfig = (env) ->
+executeSpecialActionsAndHostConfig = (conf, oldConf) ->
 	Promise.try ->
-		_.map specialActionEnvVars, (specialActionCallback, key) ->
+		_.map specialActionConfigVars, (specialActionCallback, key) ->
-			if env[key]? && specialActionCallback?
+			if conf[key]? && specialActionCallback?
 				# This makes the Special Action Envs only trigger their functions once.
-				if !_.has(executedSpecialActionEnvVars, key) or executedSpecialActionEnvVars[key] != env[key]
+				if executedSpecialActionConfigVars[key] != conf[key]
-					logSpecialAction(key, env[key])
+					logSpecialAction(key, conf[key])
-					specialActionCallback(env[key])
+					specialActionCallback(conf[key])
-					executedSpecialActionEnvVars[key] = env[key]
+					executedSpecialActionConfigVars[key] = conf[key]
-					logSpecialAction(key, env[key], true)
+					logSpecialAction(key, conf[key], true)
-		hostConfigVars = _.pick env, (val, key) ->
+		hostConfigVars = _.pick conf, (val, key) ->
-			return _.startsWith(key, device.hostConfigEnvVarPrefix)
+			return _.startsWith(key, device.hostConfigConfigVarPrefix)
-		if !_.isEmpty(hostConfigVars)
+		oldHostConfigVars = _.pick oldConf, (val, key) ->
-			device.setHostConfig(hostConfigVars, logSystemMessage)
+			return _.startsWith(key, device.hostConfigConfigVarPrefix)
+		if !_.isEqual(hostConfigVars, oldHostConfigVars)
+			device.setHostConfig(hostConfigVars, oldHostConfigVars, logSystemMessage)
+
+getAndApplyDeviceConfig = ->
+	device.getConfig()
+	.then ({ values, targetValues }) ->
+		executeSpecialActionsAndHostConfig(targetValues, values)
+		.tap ->
+			device.setConfig({ values: targetValues })
+		.then (needsReboot) ->
+			device.reboot() if needsReboot
 
 wrapAsError = (err) ->
 	return err if _.isError(err)
@@ -514,77 +505,59 @@ updateUsingStrategy = (strategy, options) ->
 		strategy = 'download-then-kill'
 	updateStrategies[strategy](options)
 
-getRemoteApps = (uuid, apiKey) ->
+getRemoteState = (uuid, apiKey) ->
-	cachedResinApi.get
+	endpoint = url.resolve(config.apiEndpoint, "/device/v1/#{uuid}/state")
-		resource: 'application'
-		options:
-			select: [
-				'id'
-				'git_repository'
-				'commit'
-			]
-			filter:
-				commit: $ne: null
-				device:
-					uuid: uuid
-		customOptions:
-			apikey: apiKey
 
-getEnvAndFormatRemoteApps = (deviceId, remoteApps, uuid, apiKey) ->
+	requestParams = _.extend
-	Promise.map remoteApps, (app) ->
+		method: 'GET'
-		getEnvironment(app.id, deviceId, apiKey)
+		url: "#{endpoint}?&apikey=#{apiKey}"
-		.then (environment) ->
+	, cachedResinApi.passthrough
-			app.environment_variable = environment
+
-			utils.extendEnvVars(app.environment_variable, uuid, app.id)
+	cachedResinApi._request(requestParams)
-		.then (fullEnv) ->
+	.catch (err) ->
-			env = _.omit(fullEnv, _.keys(specialActionEnvVars))
+		console.error("Failed to get state for device #{uuid}. #{err}")
-			env = _.omit env, (v, k) ->
+		throw err
-				_.startsWith(k, device.hostConfigEnvVarPrefix)
+
-			return [
+# TODO: Actually store and use app.environment and app.config separately
-				{
+parseEnvAndFormatRemoteApps = (remoteApps, uuid, apiKey) ->
-					appId: '' + app.id
+	appsWithEnv = _.mapValues remoteApps, (app, appId) ->
-					env: fullEnv
+		utils.extendEnvVars(app.environment, uuid, appId, app.name, app.commit)
-				},
+		.then (env) ->
-				{
+			app.config ?= {}
-					appId: '' + app.id
+			return {
-					commit: app.commit
+				appId
-					imageId: "#{config.registryEndpoint}/#{path.basename(app.git_repository, '.git')}/#{app.commit}"
+				commit: app.commit
-					env: JSON.stringify(env) # The env has to be stored as a JSON string for knex
+				imageId: app.image
-				}
+				env: JSON.stringify(env)
-			]
+				config: JSON.stringify(app.config)
-	.then(_.flatten)
+				name: app.name
-	.then(_.zip)
+			}
-	.then ([ remoteAppEnvs, remoteApps ]) ->
+	Promise.props(appsWithEnv)
-		return [_.mapValues(_.indexBy(remoteAppEnvs, 'appId'), 'env'), _.indexBy(remoteApps, 'appId')]
 
 formatLocalApps = (apps) ->
 	apps = _.indexBy(apps, 'appId')
-	localAppEnvs = {}
 	localApps = _.mapValues apps, (app) ->
-		localAppEnvs[app.appId] = JSON.parse(app.env)
+		app = _.pick(app, [ 'appId', 'commit', 'imageId', 'env', 'config', 'name' ])
-		app.env = _.omit localAppEnvs[app.appId], (v, k) ->
+	return localApps
-			_.startsWith(k, device.hostConfigEnvVarPrefix)
-		app.env = JSON.stringify(_.omit(app.env, _.keys(specialActionEnvVars)))
-		app = _.pick(app, [ 'appId', 'commit', 'imageId', 'env' ])
-	return { localApps, localAppEnvs }
 
-compareForUpdate = (localApps, remoteApps, localAppEnvs, remoteAppEnvs) ->
+compareForUpdate = (localApps, remoteApps) ->
 	remoteAppIds = _.keys(remoteApps)
 	localAppIds = _.keys(localApps)
-	appsWithChangedEnvs = _.filter remoteAppIds, (appId) ->
+
-		return !localAppEnvs[appId]? or !_.isEqual(remoteAppEnvs[appId], localAppEnvs[appId])
 	toBeRemoved = _.difference(localAppIds, remoteAppIds)
 	toBeInstalled = _.difference(remoteAppIds, localAppIds)
 
 	toBeUpdated = _.intersection(remoteAppIds, localAppIds)
 	toBeUpdated = _.filter toBeUpdated, (appId) ->
-		return !_.isEqual(remoteApps[appId], localApps[appId])
+		localApp = _.omit(localApps[appId], 'config')
+		remoteApp = _.omit(remoteApps[appId], 'config')
+		return !_.isEqual(remoteApp, localApp)
 
 	toBeDownloaded = _.filter toBeUpdated, (appId) ->
 		return !_.isEqual(remoteApps[appId].imageId, localApps[appId].imageId)
 	toBeDownloaded = _.union(toBeDownloaded, toBeInstalled)
 	allAppIds = _.union(localAppIds, remoteAppIds)
-	return { toBeRemoved, toBeDownloaded, toBeInstalled, toBeUpdated, appsWithChangedEnvs, allAppIds }
+	return { toBeRemoved, toBeDownloaded, toBeInstalled, toBeUpdated, allAppIds }
 
 application.update = update = (force) ->
 	if updateStatus.state isnt UPDATE_IDLE
@@ -594,34 +567,31 @@ application.update = update = (force) ->
 		return
 	updateStatus.state = UPDATE_UPDATING
 	bootstrap.done.then ->
-		Promise.join utils.getConfig('apiKey'), utils.getConfig('uuid'), knex('app').select(), (apiKey, uuid, apps) ->
+		Promise.join utils.getConfig('apiKey'), utils.getConfig('uuid'), utils.getConfig('name'), knex('app').select(), (apiKey, uuid, deviceName, apps) ->
-			deviceId = device.getID()
+			getRemoteState(uuid, apiKey)
-			remoteApps = getRemoteApps(uuid, apiKey)
+			.then ({ local, dependent }) ->
-
+				proxyvisor.fetchAndSetTargetsForDependentApps(dependent, fetch, apiKey)
-			Promise.join deviceId, remoteApps, uuid, apiKey, getEnvAndFormatRemoteApps
+				.then ->
-			.then ([ remoteAppEnvs, remoteApps ]) ->
+					utils.setConfig('name', deviceName) if local.name != deviceName
-				{ localApps, localAppEnvs } = formatLocalApps(apps)
+				.then ->
-				resourcesForUpdate = compareForUpdate(localApps, remoteApps, localAppEnvs, remoteAppEnvs)
+					parseEnvAndFormatRemoteApps(local.apps, uuid, apiKey)
-				{ toBeRemoved, toBeDownloaded, toBeInstalled, toBeUpdated, appsWithChangedEnvs, allAppIds } = resourcesForUpdate
+			.then (remoteApps) ->
+				localApps = formatLocalApps(apps)
+				resourcesForUpdate = compareForUpdate(localApps, remoteApps)
+				{ toBeRemoved, toBeDownloaded, toBeInstalled, toBeUpdated, allAppIds } = resourcesForUpdate
 
 				if !_.isEmpty(toBeRemoved) or !_.isEmpty(toBeInstalled) or !_.isEmpty(toBeUpdated)
 					device.setUpdateState(update_pending: true)
-				# Run special functions against variables if remoteAppEnvs has the corresponding variable function mapping.
+				# Run special functions against variables
-				Promise.map appsWithChangedEnvs, (appId) ->
+				remoteDeviceConfig = {}
-					Promise.using lockUpdates(remoteApps[appId], force), ->
+				Promise.map allAppIds, (appId) ->
-						executeSpecialActionsAndHostConfig(remoteAppEnvs[appId])
+					_.merge(remoteDeviceConfig, JSON.parse(remoteApps[appId].config))
-						.tap ->
+				.then ->
-							# If an env var shouldn't cause a restart but requires an action, we should still
+					device.setConfig({ targetValues: remoteDeviceConfig })
-							# save the new env to the DB
+					.then ->
-							if !_.includes(toBeUpdated, appId) and !_.includes(toBeInstalled, appId)
+						getAndApplyDeviceConfig()
-								utils.getKnexApp(appId)
+				.catch (err) ->
-								.then (app) ->
+					logSystemMessage("Error applying device configuration: #{err}", { error: err }, 'Set device configuration error')
-									app.env = JSON.stringify(remoteAppEnvs[appId])
-									knex('app').update(app).where({ appId })
-						.then (needsReboot) ->
-							device.reboot() if needsReboot
-					.catch (err) ->
-						logSystemEvent(logTypes.updateAppError, remoteApps[appId], err)
 				.return(allAppIds)
 				.map (appId) ->
 					Promise.try ->
@@ -639,21 +609,18 @@ application.update = update = (force) ->
 								throw err
 						else if _.includes(toBeInstalled, appId)
 							app = remoteApps[appId]
-							# Restore the complete environment so that it's persisted in the DB
-							app.env = JSON.stringify(remoteAppEnvs[appId])
 							Promise.try ->
-								fetch(remoteApps[appId]) if needsDownload
+								fetch(app) if needsDownload
 							.then ->
 								start(app)
 						else if _.includes(toBeUpdated, appId)
 							app = remoteApps[appId]
-							# Restore the complete environment so that it's persisted in the DB
+							conf = JSON.parse(app.config)
-							app.env = JSON.stringify(remoteAppEnvs[appId])
 							forceThisApp =
-								remoteAppEnvs[appId]['RESIN_SUPERVISOR_OVERRIDE_LOCK'] == '1' ||
+								conf['RESIN_SUPERVISOR_OVERRIDE_LOCK'] == '1' ||
-								remoteAppEnvs[appId]['RESIN_OVERRIDE_LOCK'] == '1'
+								conf['RESIN_OVERRIDE_LOCK'] == '1'
-							strategy = remoteAppEnvs[appId]['RESIN_SUPERVISOR_UPDATE_STRATEGY']
+							strategy = conf['RESIN_SUPERVISOR_UPDATE_STRATEGY']
-							timeout = remoteAppEnvs[appId]['RESIN_SUPERVISOR_HANDOVER_TIMEOUT']
+							timeout = conf['RESIN_SUPERVISOR_HANDOVER_TIMEOUT']
 							updateUsingStrategy strategy, {
 								localApp: localApps[appId]
 								app
@@ -666,6 +633,8 @@ application.update = update = (force) ->
 		.then (failures) ->
 			_.each(failures, (err) -> console.error('Error:', err, err.stack))
 			throw new Error(joinErrorMessages(failures)) if failures.length > 0
+		.then ->
+			proxyvisor.sendUpdates()
 		.then ->
 			updateStatus.failed = 0
 			device.setUpdateState(update_pending: false, update_downloaded: false, update_failed: false)
@@ -723,12 +692,11 @@ listenToEvents = do ->
 
 application.initialize = ->
 	listenToEvents()
-	knex('app').select()
+	getAndApplyDeviceConfig()
-	.then (apps) ->
+	.then ->
-		Promise.map apps, (app) ->
+		knex('app').select()
-			executeSpecialActionsAndHostConfig(JSON.parse(app.env))
+	.map (app) ->
-			.then ->
+		unlockAndStart(app)
-				unlockAndStart(app)
 	.catch (error) ->
 		console.error('Error starting apps:', error)
 	.then ->

src/bootstrap.coffee

@@ -20,7 +20,7 @@ loadPreloadedApps = ->
 		fs.readFileAsync(appsPath, 'utf8')
 	.then(JSON.parse)
 	.map (app) ->
-		utils.extendEnvVars(app.env, userConfig.uuid, app.appId)
+		utils.extendEnvVars(app.env, userConfig.uuid, app.appId, app.name, app.commit)
 		.then (extendedEnv) ->
 			app.env = JSON.stringify(extendedEnv)
 			knex('app').insert(app)

src/config.coffee

@@ -50,3 +50,4 @@ module.exports =
 	]
 	dataPath: checkString(process.env.RESIN_DATA_PATH) ? '/resin-data'
 	bootMountPoint: checkString(process.env.BOOT_MOUNTPOINT) ? '/boot'
+	proxyvisorHookReceiver: checkString(process.env.RESIN_PROXYVISOR_HOOK_RECEIVER) ? 'http://0.0.0.0:1337'

src/db.coffee

@@ -22,6 +22,15 @@ knex.init = Promise.all([
 				t.string('key').primary()
 				t.string('value')
 
+	knex.schema.hasTable('deviceConfig')
+	.then (exists) ->
+		if not exists
+			knex.schema.createTable 'deviceConfig', (t) ->
+				t.json('values')
+				t.json('targetValues')
+			.then ->
+				knex('deviceConfig').insert({ values: '{}', targetValues: '{}' })
+
 	knex.schema.hasTable('app')
 	.then (exists) ->
 		if not exists
@@ -34,10 +43,12 @@ knex.init = Promise.all([
 				t.string('appId')
 				t.boolean('privileged')
 				t.json('env')
+				t.json('config')
 		else
 			Promise.all [
 				addColumn('app', 'commit', 'string')
 				addColumn('app', 'appId', 'string')
+				addColumn('app', 'config', 'json')
 			]
 
 	knex.schema.hasTable('image')
@@ -53,6 +64,38 @@ knex.init = Promise.all([
 				t.increments('id').primary()
 				t.string('containerId')
 
+	knex.schema.hasTable('dependentApp')
+	.then (exists) ->
+		if not exists
+			knex.schema.createTable 'dependentApp', (t) ->
+				t.increments('id').primary()
+				t.string('appId')
+				t.string('parentAppId')
+				t.string('name')
+				t.string('commit')
+				t.string('imageId')
+				t.json('config')
+
+	knex.schema.hasTable('dependentDevice')
+	.then (exists) ->
+		if not exists
+			knex.schema.createTable 'dependentDevice', (t) ->
+				t.increments('id').primary()
+				t.string('uuid')
+				t.string('appId')
+				t.string('device_type')
+				t.string('logs_channel')
+				t.string('deviceId')
+				t.boolean('is_online')
+				t.string('name')
+				t.string('status')
+				t.string('download_progress')
+				t.string('commit')
+				t.string('targetCommit')
+				t.json('environment')
+				t.json('targetEnvironment')
+				t.json('config')
+				t.json('targetConfig')
 ])
 
 module.exports = knex

src/device.coffee

@@ -6,7 +6,6 @@ utils = require './utils'
 device = exports
 config = require './config'
 configPath = '/boot/config.json'
-request = Promise.promisifyAll(require('request'))
 execAsync = Promise.promisify(require('child_process').exec)
 fs = Promise.promisifyAll(require('fs'))
 bootstrap = require './bootstrap'
@@ -44,7 +43,7 @@ exports.getID = do ->
 exports.reboot = ->
 	utils.gosuper.postAsync('/v1/reboot')
 
-exports.hostConfigEnvVarPrefix = 'RESIN_HOST_'
+exports.hostConfigConfigVarPrefix = 'RESIN_HOST_'
 bootConfigEnvVarPrefix = 'RESIN_HOST_CONFIG_'
 bootBlockDevice = '/dev/mmcblk0p1'
 bootMountPoint = '/mnt/root' + config.bootMountPoint
@@ -76,11 +75,11 @@ parseBootConfigFromEnv = (env) ->
 	parsedEnv = _.omit(parsedEnv, forbiddenConfigKeys)
 	return parsedEnv
 
-exports.setHostConfig = (env, logMessage) ->
+exports.setHostConfig = (env, oldEnv, logMessage) ->
-	Promise.join setBootConfig(env, logMessage), setLogToDisplay(env, logMessage), (bootConfigApplied, logToDisplayChanged) ->
+	Promise.join setBootConfig(env, oldEnv, logMessage), setLogToDisplay(env, oldEnv, logMessage), (bootConfigApplied, logToDisplayChanged) ->
 		return (bootConfigApplied or logToDisplayChanged)
 
-setLogToDisplay = (env, logMessage) ->
+setLogToDisplay = (env, oldEnv, logMessage) ->
 	if env['RESIN_HOST_LOG_TO_DISPLAY']?
 		enable = env['RESIN_HOST_LOG_TO_DISPLAY'] != '0'
 		utils.gosuper.postAsync('/v1/set-log-to-display', { json: true, body: Enable: enable })
@@ -98,12 +97,12 @@ setLogToDisplay = (env, logMessage) ->
 	else
 		return Promise.resolve(false)
 
-setBootConfig = (env, logMessage) ->
+setBootConfig = (env, oldEnv, logMessage) ->
 	device.getDeviceType()
 	.then (deviceType) ->
 		throw new Error('This is not a Raspberry Pi') if !_.startsWith(deviceType, 'raspberry')
-		Promise.join parseBootConfigFromEnv(env), fs.readFileAsync(bootConfigPath, 'utf8'), (configFromApp, configTxt ) ->
+		Promise.join parseBootConfigFromEnv(env), parseBootConfigFromEnv(oldEnv), fs.readFileAsync(bootConfigPath, 'utf8'), (configFromApp, oldConfigFromApp, configTxt ) ->
-			throw new Error('No boot config to change') if _.isEmpty(configFromApp)
+			throw new Error('No boot config to change') if _.isEmpty(configFromApp) or _.isEqual(configFromApp, oldConfigFromApp)
 			configFromFS = {}
 			configPositions = []
 			configStatements = configTxt.split(/\r?\n/)
@@ -117,8 +116,10 @@ setBootConfig = (env, logMessage) ->
 					configPositions.push(configStr)
 			# configFromApp and configFromFS now have compatible formats
 			keysFromApp = _.keys(configFromApp)
+			keysFromOldConf = _.keys(oldConfigFromApp)
 			keysFromFS = _.keys(configFromFS)
 			toBeAdded = _.difference(keysFromApp, keysFromFS)
+			toBeDeleted = _.difference(keysFromOldConf, keysFromApp)
 			toBeChanged = _.intersection(keysFromApp, keysFromFS)
 			toBeChanged = _.filter toBeChanged, (key) ->
 				configFromApp[key] != configFromFS[key]
@@ -131,13 +132,13 @@ setBootConfig = (env, logMessage) ->
 				configStatement = null
 				if _.includes(toBeChanged, key)
 					configStatement = "#{key}=#{configFromApp[key]}"
-				else
+				else if !_.includes(toBeDeleted, key)
 					configStatement = configStatements[index]
 				return configStatement
 			# Here's the dangerous part:
 			execAsync("mount -t vfat -o remount,rw #{bootBlockDevice} #{bootMountPoint}")
 			.then ->
-				fs.writeFileAsync(bootConfigPath + '.new', outputConfig.join('\n'))
+				fs.writeFileAsync(bootConfigPath + '.new', _.reject(outputConfig, _.isNil).join('\n'))
 			.then ->
 				fs.renameAsync(bootConfigPath + '.new', bootConfigPath)
 			.then ->
@@ -226,3 +227,16 @@ do ->
 
 exports.getOSVersion = ->
 	return utils.getOSVersion(config.hostOsVersionPath)
+
+exports.getConfig = ->
+	knex('deviceConfig').select()
+	.then ([ deviceConfig ]) ->
+		return {
+			values: JSON.parse(deviceConfig.values)
+			targetValues: JSON.parse(deviceConfig.targetValues)
+		}
+exports.setConfig = (conf) ->
+	confToUpdate = {}
+	confToUpdate.values = JSON.stringify(conf.values) if conf.values?
+	confToUpdate.targetValues = JSON.stringify(conf.targetValues) if conf.targetValues?
+	knex('deviceConfig').update(confToUpdate)

src/docker-utils.coffee

@@ -114,16 +114,19 @@ do ->
 					return [ image.repoTag, 'docker.io/' + image.repoTag ]
 				.then(_.flatten)
 				knex('app').select()
-				.map (app) ->
+				.map ({ imageId }) ->
-					app.imageId + ':latest'
+					imageId + ':latest'
+				knex('dependentApp').select()
+				.map ({ imageId }) ->
+					imageId + ':latest'
 				docker.listImagesAsync()
-				(locallyCreatedTags, apps, images) ->
+				(locallyCreatedTags, apps, dependentApps, images) ->
 					imageTags = _.map(images, 'RepoTags')
 					supervisorTags = _.filter imageTags, (tags) ->
 						_.contains(tags, supervisorTag)
 					appTags = _.filter imageTags, (tags) ->
 						_.any tags, (tag) ->
-							_.contains(apps, tag)
+							_.contains(apps, tag) or _.contains(dependentApps, tag)
 					supervisorTags = _.flatten(supervisorTags)
 					appTags = _.flatten(appTags)
 					locallyCreatedTags = _.flatten(locallyCreatedTags)
@@ -195,7 +198,10 @@ do ->
 				else
 					docker.importImageAsync(req, { repo, tag, registry })
 			.then (stream) ->
-				stream.pipe(res)
+				new Promise (resolve, reject) ->
+					stream.on('error', reject)
+					.on('response', -> resolve())
+					.pipe(res)
 		.catch (err) ->
 			res.status(500).send(err?.message or err or 'Unknown error')
 
@@ -208,6 +214,9 @@ do ->
 			.then ->
 				dockerProgress.pull(repoTag, onProgress)
 
+	exports.getImageTarStream = (image) ->
+		docker.getImage(image).getAsync()
+
 	exports.loadImage = (req, res) ->
 		Promise.using writeLockImages(), ->
 			docker.listImagesAsync()
@@ -366,4 +375,11 @@ do ->
 		.then (data) ->
 			res.json(data)
 		.catch (err) ->
-			res.status(500).send(err?.message or err or 'Unknown error')
+			res.status(500).send(err?.message or err or 'Unknown error')
+
+	exports.getImageEnv = (id) ->
+		docker.getImage(id).inspectAsync()
+		.get('Config').get('Env')
+		.catch (err) ->
+			console.log('Error getting env from image', err, err.stack)
+			return {}

src/proxyvisor.coffee

@@ -0,0 +1,316 @@
+Promise = require 'bluebird'
+dockerUtils = require './docker-utils'
+{ docker } = dockerUtils
+express = require 'express'
+fs = Promise.promisifyAll require 'fs'
+{ resinApi } = require './request'
+knex = require './db'
+_ = require 'lodash'
+deviceRegister = require 'resin-register-device'
+randomHexString = require './lib/random-hex-string'
+utils = require './utils'
+device = require './device'
+bodyParser = require 'body-parser'
+request = Promise.promisifyAll require 'request'
+appConfig = require './config'
+PUBNUB = require 'pubnub'
+execAsync = Promise.promisify(require('child_process').exec)
+url = require 'url'
+
+pubnub = PUBNUB.init(appConfig.pubnub)
+
+getAssetsPath = (image) ->
+	docker.imageRootDir(image)
+	.then (rootDir) ->
+		return rootDir + '/assets'
+
+isDefined = _.negate(_.isUndefined)
+
+exports.router = router = express.Router()
+router.use(bodyParser())
+
+parseDeviceFields = (device) ->
+	device.id = parseInt(device.deviceId)
+	device.appId = parseInt(device.appId)
+	device.config = JSON.parse(device.config ? '{}')
+	device.environment = JSON.parse(device.environment ? '{}')
+	device.targetConfig = JSON.parse(device.targetConfig ? '{}')
+	device.targetEnvironment = JSON.parse(device.targetEnvironment ? '{}')
+	return device
+
+
+router.get '/v1/devices', (req, res) ->
+	knex('dependentDevice').select()
+	.map(parseDeviceFields)
+	.then (devices) ->
+		res.json(devices)
+	.catch (err) ->
+		res.status(503).send(err?.message or err or 'Unknown error')
+
+router.post '/v1/devices', (req, res) ->
+	Promise.join(
+		utils.getConfig('apiKey')
+		utils.getConfig('userId')
+		device.getID()
+		deviceRegister.generateUUID()
+		randomHexString.generate()
+		(apiKey, userId, deviceId, uuid, logsChannel) ->
+			d =
+				user: userId
+				application: req.body.appId
+				uuid: uuid
+				device_type: 'edge'
+				device: deviceId
+				registered_at: Math.floor(Date.now() / 1000)
+				logs_channel: logsChannel
+				status: 'Provisioned'
+			resinApi.post
+				resource: 'device'
+				body: d
+				customOptions:
+					apikey: apiKey
+			.then (dev) ->
+				deviceForDB = {
+					uuid: uuid
+					appId: d.application
+					device_type: d.device_type
+					deviceId: dev.id
+					name: dev.name
+					status: d.status
+					logs_channel: d.logs_channel
+				}
+				knex('dependentDevice').insert(deviceForDB)
+				.then ->
+					res.status(201).send(dev)
+	)
+	.catch (err) ->
+		console.error("Error on #{req.method} #{url.parse(req.url).pathname}", err, err.stack)
+		res.status(503).send(err?.message or err or 'Unknown error')
+
+router.get '/v1/devices/:uuid', (req, res) ->
+	uuid = req.params.uuid
+	knex('dependentDevice').select().where({ uuid })
+	.then ([ device ]) ->
+		return res.status(404).send('Device not found') if !device?
+		res.json(parseDeviceFields(device))
+	.catch (err) ->
+		console.error("Error on #{req.method} #{url.parse(req.url).pathname}", err, err.stack)
+		res.status(503).send(err?.message or err or 'Unknown error')
+
+router.post '/v1/devices/:uuid/logs', (req, res) ->
+	uuid = req.params.uuid
+	m = {
+		message: req.body.message
+		timestamp: req.body.timestamp or Date.now()
+	}
+	m.isSystem = req.body.isSystem if req.body.isSystem?
+
+	knex('dependentDevice').select().where({ uuid })
+	.then ([ device ]) ->
+		return res.status(404).send('Device not found') if !device?
+		pubnub.publish({ channel: "device-#{device.logs_channel}-logs", message: m })
+		res.status(202).send('OK')
+	.catch (err) ->
+		console.error("Error on #{req.method} #{url.parse(req.url).pathname}", err, err.stack)
+		res.status(503).send(err?.message or err or 'Unknown error')
+
+validStringOrUndefined = (s) ->
+	_.isUndefined(s) or !_.isEmpty(s)
+validObjectOrUndefined = (o) ->
+	_.isUndefined(o) or _.isObject(o)
+
+router.put '/v1/devices/:uuid', (req, res) ->
+	uuid = req.params.uuid
+	{ status, is_online, commit, environment, config } = req.body
+	if isDefined(is_online) and !_.isBoolean(is_online)
+		res.status(400).send('is_online must be a boolean')
+		return
+	if !validStringOrUndefined(status)
+		res.status(400).send('status must be a non-empty string')
+		return
+	if !validStringOrUndefined(commit)
+		res.status(400).send('commit must be a non-empty string')
+		return
+	if !validObjectOrUndefined(environment)
+		res.status(400).send('environment must be an object')
+		return
+	if !validObjectOrUndefined(config)
+		res.status(400).send('config must be an object')
+		return
+	environment = JSON.stringify(environment) if isDefined(environment)
+	config = JSON.stringify(config) if isDefined(config)
+
+	Promise.join(
+		utils.getConfig('apiKey')
+		knex('dependentDevice').select().where({ uuid })
+		(apiKey, [ device ]) ->
+			throw new Error('apikey not found') if !apiKey?
+			return res.status(404).send('Device not found') if !device?
+			resinApi.patch
+				resource: 'device'
+				id: device.deviceId
+				body: _.pick({ status, is_online, commit }, isDefined)
+				customOptions:
+					apikey: apiKey
+			.then ->
+				fieldsToUpdate = _.pick({ status, is_online, commit, config, environment }, isDefined)
+				knex('dependentDevice').update(fieldsToUpdate).where({ uuid })
+			.then ->
+				res.json(parseDeviceFields(device))
+	)
+	.catch (err) ->
+		console.error("Error on #{req.method} #{url.parse(req.url).pathname}", err, err.stack)
+		res.status(503).send(err?.message or err or 'Unknown error')
+
+tarPath = ({ commit }) ->
+	return '/tmp/' + commit + '.tar'
+
+router.get '/v1/dependent-apps/:appId/assets/:commit', (req, res) ->
+	knex('dependentApp').select().where(_.pick(req.params, 'appId', 'commit'))
+	.then ([ app ]) ->
+		return res.status(404).send('Not found') if !app
+		dest = tarPath(app)
+		getAssetsPath(app.imageId)
+		.then (path) ->
+			getTarArchive(path, dest)
+		.then ->
+			res.sendFile(dest)
+	.catch (err) ->
+		console.error("Error on #{req.method} #{url.parse(req.url).pathname}", err, err.stack)
+		res.status(503).send(err?.message or err or 'Unknown error')
+
+router.get '/v1/dependent-apps', (req, res) ->
+	knex('dependentApp').select()
+	.map (app) ->
+		return {
+			id: parseInt(app.appId)
+			commit: app.commit
+			device_type: 'edge'
+			name: app.name
+			config: JSON.parse(app.config ? '{}')
+		}
+	.then (apps) ->
+		res.json(apps)
+	.catch (err) ->
+		console.error("Error on #{req.method} #{url.parse(req.url).pathname}", err, err.stack)
+		res.status(503).send(err?.message or err or 'Unknown error')
+
+getTarArchive = (path, destination) ->
+	fs.lstatAsync(path)
+	.then ->
+		execAsync("tar -cvf '#{destination}' *", cwd: path)
+
+# TODO: deduplicate code from compareForUpdate in application.coffee
+exports.fetchAndSetTargetsForDependentApps = (state, fetchFn, apiKey) ->
+	knex('dependentApp').select()
+	.then (localDependentApps) ->
+		# Compare to see which to fetch, and which to delete
+		remoteApps = _.mapValues state.apps, (app, appId) ->
+			conf = app.config ? {}
+			return {
+				appId: appId
+				parentAppId: app.parentApp
+				imageId: app.image
+				commit: app.commit
+				config: JSON.stringify(conf)
+				name: app.name
+			}
+		localApps = _.indexBy(localDependentApps, 'appId')
+
+		toBeDownloaded = _.filter remoteApps, (app, appId) ->
+			return app.commit? and app.imageId? and !_.any(localApps, imageId: app.imageId)
+		toBeRemoved = _.filter localApps, (app, appId) ->
+			return app.commit? and !_.any(remoteApps, imageId: app.imageId)
+		Promise.map toBeDownloaded, (app) ->
+			fetchFn(app, false)
+		.then ->
+			Promise.map toBeRemoved, (app) ->
+				fs.unlinkAsync(tarPath(app))
+				.then ->
+					docker.getImage(app.imageId).removeAsync()
+				.catch (err) ->
+					console.error('Could not remove image/artifacts for dependent app', err, err.stack)
+		.then ->
+			Promise.props(
+				_.mapValues remoteApps, (app, appId) ->
+					knex('dependentApp').update(app).where({ appId })
+					.then (n) ->
+						knex('dependentApp').insert(app) if n == 0
+			)
+		.then ->
+			Promise.all _.map state.devices, (device, uuid) ->
+				# Only consider one app per dependent device for now
+				appId = _(device.apps).keys().first()
+				targetCommit = state.apps[appId].commit
+				targetEnvironment = JSON.stringify(device.apps[appId].environment ? {})
+				targetConfig = JSON.stringify(device.apps[appId].config ? {})
+				knex('dependentDevice').update({ targetEnvironment, targetConfig, targetCommit, name: device.name }).where({ uuid })
+				.then (n) ->
+					return if n != 0
+					# If the device is not in the DB it means it was provisioned externally
+					# so we need to fetch it.
+					resinApi.get
+						resource: 'device'
+						options:
+							filter:
+								uuid: uuid
+						customOptions:
+							apikey: apiKey
+					.then ([ dev ]) ->
+						deviceForDB = {
+							uuid: uuid
+							appId: appId
+							device_type: dev.device_type
+							deviceId: dev.id
+							is_online: dev.is_online
+							name: dev.name
+							status: dev.status
+							logs_channel: dev.logs_channel
+							targetCommit
+							targetConfig
+							targetEnvironment
+						}
+						knex('dependentDevice').insert(deviceForDB)
+	.catch (err) ->
+		console.error('Error fetching dependent apps', err, err.stack)
+
+sendUpdate = (device, endpoint) ->
+	request.putAsync "#{endpoint}#{device.uuid}", {
+		json: true
+		body:
+			appId: device.appId
+			commit: device.targetCommit
+			environment: JSON.parse(device.targetEnvironment)
+			config: JSON.parse(device.targetConfig)
+	}
+	.spread (response, body) ->
+		if response.statusCode != 200
+			return console.error("Error updating device #{device.uuid}: #{response.statusCode} #{body}")
+
+getHookEndpoint = (appId) ->
+	knex('dependentApp').select('parentAppId').where({ appId })
+	.then ([ { parentAppId } ]) ->
+		knex('app').select().where({ appId: parentAppId })
+	.then ([ parentApp ]) ->
+		conf = JSON.parse(parentApp.config)
+		dockerUtils.getImageEnv(parentApp.imageId)
+		.then (imageEnv) ->
+			return imageEnv.RESIN_DEPENDENT_DEVICES_HOOK_ADDRESS ?
+				conf.RESIN_DEPENDENT_DEVICES_HOOK_ADDRESS ?
+				"#{appConfig.proxyvisorHookReceiver}/v1/devices/"
+
+exports.sendUpdates = ->
+	endpoints = {}
+	knex('dependentDevice').select()
+	.map (device) ->
+		currentState = _.pick(device, 'commit', 'environment', 'config')
+		targetState = {
+			commit: device.targetCommit
+			environment: device.targetEnvironment
+			config: device.targetConfig
+		}
+		if device.targetCommit? and !_.isEqual(targetState, currentState)
+			endpoints[device.appId] ?= getHookEndpoint(device.appId)
+			endpoints[device.appId]
+			.then (endpoint) ->
+				sendUpdate(device, endpoint)

src/request.coffee

@@ -8,7 +8,7 @@ requestOpts =
 	gzip: true
 	timeout: 30000
 
-PLATFORM_ENDPOINT = url.resolve(config.apiEndpoint, '/ewa/')
+PLATFORM_ENDPOINT = url.resolve(config.apiEndpoint, '/v2/')
 exports.resinApi = resinApi = new PlatformAPI
 	apiPrefix: PLATFORM_ENDPOINT
 	passthrough: requestOpts

src/utils.coffee

@@ -12,6 +12,7 @@ request = Promise.promisifyAll require 'request'
 logger = require './lib/logger'
 TypedError = require 'typed-error'
 execAsync = Promise.promisify(require('child_process').exec)
+device = require './device'
 
 # Parses package.json and returns resin-supervisor's version
 version = require('../package.json').version
@@ -149,11 +150,21 @@ exports.getConfig = getConfig = (key) ->
 	.then ([ conf ]) ->
 		return conf?.value
 
-exports.extendEnvVars = (env, uuid, appId) ->
+exports.setConfig = (key, value) ->
+	knex('config').update({ value }).where({ key })
+	.then (n) ->
+		knex('config').insert({ key, value }) if n == 0
+
+exports.extendEnvVars = (env, uuid, appId, appName, commit) ->
 	host = '127.0.0.1'
 	newEnv =
 		RESIN_APP_ID: appId.toString()
+		RESIN_APP_NAME: appName
+		RESIN_APP_RELEASE: commit
 		RESIN_DEVICE_UUID: uuid
+		RESIN_DEVICE_NAME_AT_INIT: getConfig('name')
+		RESIN_DEVICE_TYPE: device.getDeviceType()
+		RESIN_HOST_OS_VERSION: device.getOSVersion()
 		RESIN_SUPERVISOR_ADDRESS: "http://#{host}:#{config.listenPort}"
 		RESIN_SUPERVISOR_HOST: host
 		RESIN_SUPERVISOR_PORT: config.listenPort