Refactor bootstrapper. Run preloaded supervisor dind. Change dind configs to be ignored and document how to populate them.

.gitignore

@@ -3,3 +3,6 @@
 data
 bin/gosuper
 gosuper/bin/
+tools/dind/config.json
+tools/dind/apps.json
+tools/dind/config/localenv

Makefile

@@ -10,6 +10,7 @@ JOB_NAME = 1
 all: supervisor 
 
 IMAGE = "resin/$(ARCH)-supervisor:$(SUPERVISOR_VERSION)"
+SUPERVISOR_IMAGE=$(DEPLOY_REGISTRY)$(IMAGE)
 
 ifeq ($(ARCH),rpi)
 	GOARCH = arm
@@ -23,6 +24,12 @@ endif
 ifeq ($(ARCH),amd64)
 	GOARCH = amd64
 endif
+SUPERVISOR_DIND_MOUNTS := -v $$(pwd)/config.json:/mnt/conf/config.json -v $$(pwd)/config/env:/usr/src/app/config/env -v $$(pwd)/config/localenv:/usr/src/app/config/localenv -v /sys/fs/cgroup:/sys/fs/cgroup:ro
+ifdef PRELOADED_IMAGE
+	SUPERVISOR_DIND_MOUNTS := ${SUPERVISOR_DIND_MOUNTS} -v $$(pwd)/apps.json:/usr/src/app/config/apps.json
+else
+	PRELOADED_IMAGE=
+endif
 
 clean:
 	-rm Dockerfile
@@ -32,8 +39,8 @@ supervisor-dind:
 
 run-supervisor: supervisor-dind stop-supervisor
 	cd tools/dind \
-	&& sed --in-place -e "s|SUPERVISOR_IMAGE=.*|SUPERVISOR_IMAGE=$(DEPLOY_REGISTRY)$(IMAGE) |" config/env \
-	&& docker run -d --name resin_supervisor_1 --privileged -v $$(pwd)/config.json:/mnt/conf/config.json -v $$(pwd)/config/env:/usr/src/app/config/env -v /sys/fs/cgroup:/sys/fs/cgroup:ro resin/resin-supervisor-dind:$(SUPERVISOR_VERSION)
+	&& echo "SUPERVISOR_IMAGE=$(SUPERVISOR_IMAGE)\nPRELOADED_IMAGE=$(PRELOADED_IMAGE)" > config/localenv \
+	&& docker run -d --name resin_supervisor_1 --privileged ${SUPERVISOR_DIND_MOUNTS} resin/resin-supervisor-dind:$(SUPERVISOR_VERSION)
 
 stop-supervisor:
 	# Stop docker and remove volumes to prevent us from running out of loopback devices,

README.md

@@ -13,17 +13,35 @@ This will build the image if you haven't done it yet.
 A different registry can be specified with the DEPLOY_REGISTRY env var.
 
 ## Set up config
-Edit `tools/dind/config.json` to contain the values for a staging config.json.
+Add `tools/dind/config.json` file from a staging device image.
 
-This file can be obtained in several ways, for instance:
+A config.json file can be obtained in several ways, for instance:
 
 * Download an Intel Edison image from staging, open `config.img` with an archive tool like [peazip](http://sourceforge.net/projects/peazip/files/)
 * Download a Raspberry Pi 2 image, flash it to an SD card, then mount partition 5 (resin-conf).
 
-Tip: to avoid git marking config.json as modified, you can run:
-```bash
-git update-index --assume-unchanged tools/dind/config.json
+The config.json file should look something like this (beautified and commented for better explanation):
+```json
+{
+	"applicationId": "2167", /* Id of the app this supervisor will run */
+	"apiKey": "supersecretapikey", /* The API key for the Resin API */
+	"userId": "141", /* User ID for the user who owns the app */
+	"username": "gh_pcarranzav", /* User name for the user who owns the app */
+	"deviceType": "intel-edison", /* The device type corresponding to the test application */
+	"files": { /* This field is used by the host OS so the supervisor doesn't care about it */
+		"network/settings": "[global]\nOfflineMode=false\n\n[WiFi]\nEnable=true\nTethering=false\n\n[Wired]\nEnable=true\nTethering=false\n\n[Bluetooth]\nEnable=true\nTethering=false",
+		"network/network.config": "[service_home_ethernet]\nType = ethernet\nNameservers = 8.8.8.8,8.8.4.4"
+	},
+	"apiEndpoint": "https://api.resinstaging.io", /* Endpoint for the Resin API */
+	"registryEndpoint": "registry.resinstaging.io", /* Endpoint for the Resin registry */
+	"vpnEndpoint": "vpn.resinstaging.io", /* Endpoint for the Resin VPN server */
+	"pubnubSubscribeKey": "sub-c-aaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa", /* Subscribe key for Pubnub for logs */
+	"pubnubPublishKey": "pub-c-aaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa", /* Publish key for Pubnub for logs */
+	"listenPort": 48484, /* Listen port for the supervisor API */
+	"mixpanelToken": "aaaaaaaaaaaaaaaaaaaaaaaaaa", /* Mixpanel token to report events */
+}
 ```
+Additionally, the `uuid`, `registered_at` and `deviceId` fields will be added by the supervisor upon registration with the resin API.
 
 ## Start the supervisor instance
 ```bash
@@ -41,6 +59,33 @@ make ARCH=amd64 DEPLOY_REGISTRY= run-supervisor
 ```
 to pull the jenkins built images from the docker hub.
 
+## Testing with preloaded apps
+To test preloaded apps, add a `tools/dind/apps.json` file according to the preloaded apps spec.
+
+It should look something like this:
+
+```json
+[{
+	"appId": "2167", /* Id of the app we are running */
+	"commit": "commithash", /* Current git commit for the app */
+	"imageId": "registry.resinstaging.io/appname/commithash", /* Id of the docker image for this app and commit */
+	"env": { /* Environment variables for the app */
+		"KEY": "value"
+	}
+}]
+```
+where `appname` and `commithash` correspond to the name of the test app and the last commit pushed to Resin.
+
+For instance, `imageId` could be `"registry.resinstaging.io/supertest/5a5f999fde38590d4c28ac80779f3999c12fd9ae"`
+
+Make sure the config.json file doesn't have uuid, registered_at or deviceId populated from a previous run.
+
+Then run the supervisor like this:
+```bash
+make ARCH=amd64 PRELOADED_IMAGE=registry.resinstaging.io/appname/commithash run-supervisor
+```
+This will make the docker-in-docker instance pull the image before running the supervisor.
+
 ## View the containers logs
 ```bash
 logs supervisor -f

src/api.coffee

@@ -1,7 +1,6 @@
 Promise = require 'bluebird'
 fs = Promise.promisifyAll require 'fs'
 utils = require './utils'
-application = require './application'
 tty = require './lib/tty'
 knex = require './db'
 express = require 'express'
@@ -9,7 +8,7 @@ bodyParser = require 'body-parser'
 request = require 'request'
 config = require './config'
 
-module.exports = (secret) ->
+module.exports = (secret, application) ->
 	api = express()
 	api.use(bodyParser())
 	api.use (req, res, next) ->

src/app.coffee

@@ -26,7 +26,7 @@ knex.init.then ->
 
 		bootstrap.done.then ->
 			console.log('Starting API server..')
-			api(secret).listen(config.listenPort)
+			api(secret, application).listen(config.listenPort)
 			# Let API know what version we are, and our api connection info.
 			console.log('Updating supervisor version and api info')
 			device.updateState(

src/application.coffee

@@ -256,6 +256,8 @@ application.unlockAndStart = unlockAndStart = (app) ->
 	.then ->
 		start(app)
 
+ENOENT = (err) -> err.code is 'ENOENT'
+
 application.lockUpdates = lockUpdates = do ->
 	_lock = new Lock()
 	_writeLock = Promise.promisify(_lock.async.writeLock)
@@ -265,6 +267,7 @@ application.lockUpdates = lockUpdates = do ->
 		.tap (release) ->
 			if force != true
 				lockFile.lockAsync(lockName)
+				.catch ENOENT, _.noop
 				.catch (err) ->
 					release()
 					throw new Error("Updates are locked: #{err.message}")

src/bootstrap.coffee

@@ -6,97 +6,97 @@ deviceRegister = require 'resin-register-device'
 { resinApi } = require './request'
 fs = Promise.promisifyAll(require('fs'))
 EventEmitter = require('events').EventEmitter
+config = require './config'
+configPath = '/boot/config.json'
+appsPath  = '/boot/apps.json'
+userConfig = {}
 
-module.exports = do ->
-	configPath = '/boot/config.json'
-	appsPath  = '/boot/apps.json'
-	userConfig = {}
+DuplicateUuidError = (err) ->
+	return err.message == '"uuid" must be unique.'
 
-	bootstrapper = new EventEmitter()
+bootstrapper = {}
 
-	loadPreloadedApps = ->
-		knex('app').truncate()
-		.then ->
-			fs.readFileAsync(appsPath, 'utf8')
-		.then(JSON.parse)
-		.then (apps) ->
-			Promise.map apps, (app) ->
-				app.env = JSON.stringify(app.env)
-				knex('app').insert(app)
-		.catch (err) ->
-			utils.mixpanelTrack('Loading preloaded apps failed', {error: err})
+loadPreloadedApps = ->
+	knex('app').truncate()
+	.then ->
+		fs.readFileAsync(appsPath, 'utf8')
+	.then(JSON.parse)
+	.map (app) ->
+		app.env = JSON.stringify(app.env)
+		knex('app').insert(app)
+	.catch (err) ->
+		utils.mixpanelTrack('Loading preloaded apps failed', {error: err})
 
-	bootstrap = ->
-		Promise.try ->
-			userConfig.deviceType ?= 'raspberry-pi'
-			if userConfig.registered_at?
-				return userConfig
-			deviceRegister.register(resinApi, userConfig)
-			.catch (err) ->
-				# Do not fail if device already exists
-				return {} if err.message = '"uuid" must be unique.'
-			.then (device) ->
-				userConfig.registered_at = Date.now()
-				userConfig.deviceId = device.id if device.id?
-				fs.writeFileAsync(configPath, JSON.stringify(userConfig))
-			.return(userConfig)
-		.then (userConfig) ->
-			console.log('Finishing bootstrapping')
-			Promise.all([
-				knex('config').truncate()
-				.then ->
-					knex('config').insert([
-						{ key: 'uuid', value: userConfig.uuid }
-						{ key: 'apiKey', value: userConfig.apiKey }
-						{ key: 'username', value: userConfig.username }
-						{ key: 'userId', value: userConfig.userId }
-						{ key: 'version', value: utils.supervisorVersion }
-					])
-			])
-			.tap ->
-				bootstrapper.doneBootstrapping()
-
-	readConfigAndEnsureUUID = ->
-		# Load config file
-		fs.readFileAsync(configPath, 'utf8')
-		.then(JSON.parse)
-		.then (config) ->
-			userConfig = config
-			return userConfig.uuid if userConfig.uuid?
-			deviceRegister.generateUUID()
-			.then (uuid) ->
-				userConfig.uuid = uuid
-				fs.writeFileAsync(configPath, JSON.stringify(userConfig))
-			.return(userConfig.uuid)
-		.catch (err) ->
-			console.log('Error generating and saving UUID: ', err)
-			Promise.delay(config.bootstrapRetryDelay)
+bootstrap = ->
+	Promise.try ->
+		userConfig.deviceType ?= 'raspberry-pi'
+		if userConfig.registered_at?
+			return userConfig
+		deviceRegister.register(resinApi, userConfig)
+		.catch DuplicateUuidError, ->
+			return {}
+		.then (device) ->
+			userConfig.registered_at = Date.now()
+			userConfig.deviceId = device.id if device.id?
+			fs.writeFileAsync(configPath, JSON.stringify(userConfig))
+		.return(userConfig)
+	.then (userConfig) ->
+		console.log('Finishing bootstrapping')
+		Promise.all([
+			knex('config').truncate()
 			.then ->
-				readConfigAndEnsureUUID()
+				knex('config').insert([
+					{ key: 'uuid', value: userConfig.uuid }
+					{ key: 'apiKey', value: userConfig.apiKey }
+					{ key: 'username', value: userConfig.username }
+					{ key: 'userId', value: userConfig.userId }
+					{ key: 'version', value: utils.supervisorVersion }
+				])
+		])
+		.tap ->
+			bootstrapper.doneBootstrapping()
 
-	bootstrapOrRetry = ->
-		utils.mixpanelTrack('Device bootstrap')
-		bootstrap().catch (err) ->
-			utils.mixpanelTrack('Device bootstrap failed, retrying', {error: err, delay: config.bootstrapRetryDelay})
-			setTimeout(bootstrapOrRetry, config.bootstrapRetryDelay)
-
-	bootstrapper.done = new Promise (resolve) ->
-		bootstrapper.doneBootstrapping = ->
-			bootstrapper.bootstrapped = true
-			resolve(userConfig)
-
-	bootstrapper.bootstrapped = false
-	bootstrapper.startBootstrapping = ->
-		knex('config').select('value').where(key: 'uuid')
-		.then ([ uuid ]) ->
-			if uuid?.value
-				bootstrapper.doneBootstrapping()
-				return uuid.value
-			console.log('New device detected. Bootstrapping..')
+readConfigAndEnsureUUID = ->
+	# Load config file
+	fs.readFileAsync(configPath, 'utf8')
+	.then(JSON.parse)
+	.then (configFromFile) ->
+		userConfig = configFromFile
+		return userConfig.uuid if userConfig.uuid?
+		deviceRegister.generateUUID()
+		.then (uuid) ->
+			userConfig.uuid = uuid
+			fs.writeFileAsync(configPath, JSON.stringify(userConfig))
+			.return(uuid)
+	.catch (err) ->
+		console.log('Error generating and saving UUID: ', err)
+		Promise.delay(config.bootstrapRetryDelay)
+		.then ->
 			readConfigAndEnsureUUID()
-			.tap ->
-				loadPreloadedApps()
-			.tap ->
-				bootstrapOrRetry()
 
-	return bootstrapper
+bootstrapOrRetry = ->
+	utils.mixpanelTrack('Device bootstrap')
+	bootstrap().catch (err) ->
+		utils.mixpanelTrack('Device bootstrap failed, retrying', {error: err, delay: config.bootstrapRetryDelay})
+		setTimeout(bootstrapOrRetry, config.bootstrapRetryDelay)
+
+bootstrapper.done = new Promise (resolve) ->
+	bootstrapper.doneBootstrapping = ->
+		bootstrapper.bootstrapped = true
+		resolve(userConfig)
+
+bootstrapper.bootstrapped = false
+bootstrapper.startBootstrapping = ->
+	knex('config').select('value').where(key: 'uuid')
+	.then ([ uuid ]) ->
+		if uuid?.value
+			bootstrapper.doneBootstrapping()
+			return uuid.value
+		console.log('New device detected. Bootstrapping..')
+		readConfigAndEnsureUUID()
+		.tap ->
+			loadPreloadedApps()
+		.tap ->
+			bootstrapOrRetry()
+
+module.exports = bootstrapper

src/device.coffee

@@ -45,15 +45,15 @@ exports.updateState = do ->
 			actualState[key] is value
 
 	applyState = ->
-		if _.size(getStateDiff()) is 0
+		stateDiff = getStateDiff()
+		if _.size(stateDiff) is 0
 			return
-
 		applyPromise = Promise.join(
 			knex('config').select('value').where(key: 'apiKey')
 			device.getID()
 			([{value: apiKey}], deviceID) ->
 				stateDiff = getStateDiff()
-				if _.size(stateDiff) is 0
+				if _.size(stateDiff) is 0 || !apiKey?
 					return
 				resinApi.patch
 					resource: 'device'
@@ -64,11 +64,11 @@ exports.updateState = do ->
 				.then ->
 					# Update the actual state.
 					_.merge(actualState, stateDiff)
-				.catch (error) ->
-					utils.mixpanelTrack('Device info update failure', {error, stateDiff})
-					# Delay 5s before retrying a failed update
-					Promise.delay(5000)
 		)
+		.catch (error) ->
+			utils.mixpanelTrack('Device info update failure', {error, stateDiff})
+			# Delay 5s before retrying a failed update
+			Promise.delay(5000)
 		.finally ->
 			# Check if any more state diffs have appeared whilst we've been processing this update.
 			applyState()

tools/dind/config/env

@@ -1,4 +1,4 @@
 CONFIG_PATH=/mnt/conf/config.json
 LED_FILE=/dev/null
 RESIN_SUPERVISOR_SECRET=bananas
-SUPERVISOR_IMAGE=
+APPS_PATH=/usr/src/app/config/apps.json

tools/dind/config/services/resin-supervisor-dind.service

@@ -7,6 +7,8 @@ Before=openvpn@client.service
 [Service]
 WorkingDirectory=/usr/src/app
 EnvironmentFile=/usr/src/app/config/env
+EnvironmentFile=/usr/src/app/config/localenv
+ExecStartPre=/bin/bash -c 'if [ -n "${PRELOADED_IMAGE}" ]; then /usr/bin/docker pull ${PRELOADED_IMAGE}; fi'
 ExecStartPre=/usr/bin/docker pull ${SUPERVISOR_IMAGE}
 ExecStartPre=-/usr/bin/docker kill resin_supervisor
 ExecStartPre=-/usr/bin/docker rm resin_supervisor
@@ -16,6 +18,7 @@ ExecStart=/bin/bash -c 'source /usr/src/app/resin-vars && \
 	--net=host \
 	-v /var/run/docker.sock:/run/docker.sock \
 	-v "${CONFIG_PATH}:/boot/config.json" \
+	-v "${APPS_PATH}:/boot/apps.json" \
 	-v /resin-data/resin-supervisor:/data \
 	-v /proc/net/fib_trie:/mnt/fib_trie \
 	-v /var/log/supervisor-log:/var/log \