Auto-merge for PR #448 via VersionBot

Remove the undocumented and unused sideload and compose APIs

CHANGELOG.md

@@ -4,6 +4,10 @@ All notable changes to this project will be documented in this file
 automatically by Versionist. DO NOT EDIT THIS FILE MANUALLY!
 This project adheres to [Semantic Versioning](http://semver.org/).
 
+## v5.0.0 - 2017-06-26
+
+* Remove the undocumented and unused sideload and compose APIs [Pablo Carranza Velez]
+
 ## v4.5.0 - 2017-06-26
 
 * Update docker-delta to 1.0.3 to support docker 17 [Pablo Carranza Velez]

Dockerfile.build.template

@@ -16,27 +16,8 @@ RUN apt-get update \
 		wget \
 	&& rm -rf /var/lib/apt/lists/
 
-ENV DOCKER_COMPOSE_VERSION 1.7.1
+RUN mkdir -p rootfs-overlay && \
-
+	ln -s /lib rootfs-overlay/lib64
-ENV DOCKER_COMPOSE_SHA256_amd64 37df85ee18bf0e2a8d71cbfb8198b1c06cc388f19118be7bdfc4d6db112af834
-ENV DOCKER_COMPOSE_SHA256_i386 b926fd9a2a9d89358f1353867706f94558a62caaf3aa72bf10bcbbe31e1a44f0
-ENV DOCKER_COMPOSE_SHA256_rpi 3f0b8c69c66a2daa5fbb0c127cb76ca95d7125827a9c43dd3c36f9bc2ed6e0e5
-ENV DOCKER_COMPOSE_SHA256_armv7hf 3f0b8c69c66a2daa5fbb0c127cb76ca95d7125827a9c43dd3c36f9bc2ed6e0e5
-ENV DOCKER_COMPOSE_SHA256_armel a1025fed97536e2698798ea277a014ec5e1eae816a8cf3155ecbe9679e3e7bac
-
-RUN set -x \
-	&& mkdir -p rootfs-overlay/usr/bin/ \
-	&& ln -s /lib rootfs-overlay/lib64 \
-	&& pkgname='docker-compose' \
-	&& arch=%%ARCH%% \
-	&& if [ $arch = 'rpi' -o $arch = 'armv7hf' ]; then arch=armhf; fi \
-	&& base="http://resin-packages.s3.amazonaws.com/${pkgname}" \
-	&& pkgver=$DOCKER_COMPOSE_VERSION \
-	&& checksum=$DOCKER_COMPOSE_SHA256_%%ARCH%% \
-	&& wget "${base}/${pkgver}/${pkgname}-linux-${arch}-${pkgver}.tar.gz" \
-	&& echo "$checksum  ${pkgname}-linux-${arch}-${pkgver}.tar.gz" | sha256sum -c \
-	&& tar xzf "${pkgname}-linux-${arch}-${pkgver}.tar.gz" --strip-components=1 -C rootfs-overlay/usr/bin \
-	&& mv "rootfs-overlay/usr/bin/${pkgname}-linux-${arch}" rootfs-overlay/usr/bin/docker-compose
 
 COPY package.json /usr/src/app/
 

package.json

@@ -1,7 +1,7 @@
 {
   "name": "resin-supervisor",
   "description": "This is resin.io's Supervisor, a program that runs on IoT devices and has the task of running user Apps (which are Docker containers), and updating them as Resin's API informs it to.",
-  "version": "4.5.0",
+  "version": "5.0.0",
   "license": "Apache-2.0",
   "repository": {
     "type": "git",
@@ -31,7 +31,6 @@
     "log-timestamp": "^0.1.2",
     "memoizee": "^0.4.1",
     "mixpanel": "0.0.20",
-    "mkdirp": "^0.5.1",
     "network-checker": "~0.0.5",
     "pinejs-client": "^2.4.0",
     "pubnub": "^3.7.13",
@@ -43,11 +42,10 @@
     "semver": "^5.3.0",
     "semver-regex": "^1.0.0",
     "sqlite3": "^3.1.0",
-    "typed-error": "~0.1.0",
+    "typed-error": "~0.1.0"
-    "yamljs": "^0.2.7"
   },
   "engines": {
-    "node": "0.10.22"
+    "node": "^6.5.0"
   },
   "devDependencies": {
     "coffee-script": "~1.11.0",

src/api.coffee

@@ -5,9 +5,7 @@ bodyParser = require 'body-parser'
 bufferEq = require 'buffer-equal-constant-time'
 config = require './config'
 device = require './device'
-dockerUtils = require './docker-utils'
 _ = require 'lodash'
-compose = require './compose'
 proxyvisor = require './proxyvisor'
 
 module.exports = (application) ->
@@ -217,53 +215,6 @@ module.exports = (application) ->
 	unparsedRouter.get '/v1/device', (req, res) ->
 		res.json(device.getState())
 
-	unparsedRouter.post '/v1/images/create', dockerUtils.createImage
-	unparsedRouter.post '/v1/images/load', dockerUtils.loadImage
-	unparsedRouter.delete '/v1/images/*', dockerUtils.deleteImage
-	unparsedRouter.get '/v1/images', dockerUtils.listImages
-	parsedRouter.post '/v1/containers/create', dockerUtils.createContainer
-	parsedRouter.post '/v1/containers/update', dockerUtils.updateContainer
-	parsedRouter.post '/v1/containers/:id/start', dockerUtils.startContainer
-	unparsedRouter.post '/v1/containers/:id/stop', dockerUtils.stopContainer
-	unparsedRouter.delete '/v1/containers/:id', dockerUtils.deleteContainer
-	unparsedRouter.get '/v1/containers', dockerUtils.listContainers
-
-	unparsedRouter.post '/v1/apps/:appId/compose/up', (req, res) ->
-		appId = req.params.appId
-		onStatus = (status) ->
-			status = JSON.stringify(status) if _.isObject(status)
-			res.write(status)
-		utils.getKnexApp(appId)
-		.then (app) ->
-			res.status(200)
-			compose.up(appId, onStatus)
-			.catch (err) ->
-				console.log('Error on compose up:', err, err.stack)
-			.finally ->
-				res.end()
-		.catch utils.AppNotFoundError, (e) ->
-			return res.status(400).send(e.message)
-		.catch (err) ->
-			res.status(503).send(err?.message or err or 'Unknown error')
-
-	unparsedRouter.post '/v1/apps/:appId/compose/down', (req, res) ->
-		appId = req.params.appId
-		onStatus = (status) ->
-			status = JSON.stringify(status) if _.isObject(status)
-			res.write(status)
-		utils.getKnexApp(appId)
-		.then (app) ->
-			res.status(200)
-			compose.down(appId, onStatus)
-			.catch (err) ->
-				console.log('Error on compose down:', err, err.stack)
-			.finally ->
-				res.end()
-		.catch utils.AppNotFoundError, (e) ->
-			return res.status(400).send(e.message)
-		.catch (err) ->
-			res.status(503).send(err?.message or err or 'Unknown error')
-
 	api.use(unparsedRouter)
 	api.use(parsedRouter)
 	api.use(proxyvisor.router)

src/compose.coffee

@@ -1,88 +0,0 @@
-Promise = require 'bluebird'
-YAML = require 'yamljs'
-_ = require 'lodash'
-dockerUtils = require './docker-utils'
-{ docker } = dockerUtils
-fs = Promise.promisifyAll(require('fs'))
-{ spawn, execAsync } = Promise.promisifyAll(require('child_process'))
-mkdirp = Promise.promisify(require('mkdirp'))
-path = require 'path'
-utils = require './utils'
-config = require './config'
-
-composePathSrc = (appId) ->
-	return "/mnt/root#{config.dataPath}/#{appId}/docker-compose.yml"
-
-composePathDst = (appId) ->
-	return "/mnt/root#{config.dataPath}/resin-supervisor/compose/#{appId}/docker-compose.yml"
-
-composeDataPath = (appId, serviceName) ->
-	return "compose/#{appId}/#{serviceName}"
-
-runComposeCommand = (composeArgs, appId, reportStatus) ->
-	new Promise (resolve, reject) ->
-		child = spawn('docker-compose', ['-f', composePathDst(appId)].concat(composeArgs), stdio: 'pipe')
-		.on 'error', reject
-		.on 'exit', (code) ->
-			return reject(new Error("docker-compose exited with code #{code}")) if code isnt 0
-			resolve()
-		child.stdout.on 'data', (data) ->
-			reportStatus(status: '' + data)
-		child.stderr.on 'data', (data) ->
-			reportStatus(status: '' + data)
-
-writeComposeFile = (composeSpec, dstPath) ->
-	mkdirp(path.dirname(dstPath))
-	.then ->
-		YAML.stringify(composeSpec)
-	.then (yml) ->
-		fs.writeFileAsync(dstPath, yml)
-	.then ->
-		execAsync('sync')
-
-# Runs docker-compose up using the compose YAML at "path".
-# Reports status and errors in JSON to the onStatus function.
-# Copies the compose file from srcPath to dstPath adding default volumes
-exports.up = (appId, onStatus) ->
-	onStatus ?= console.log.bind(console)
-	reportStatus = (status) ->
-		try onStatus(status)
-	fs.readFileAsync(composePathSrc(appId))
-	.then (data) ->
-		YAML.parse(data.toString())
-	.then (composeSpec) ->
-		if composeSpec.version? && composeSpec.version == '2'
-			services = composeSpec.services
-		else
-			services = composeSpec
-		throw new Error('No services found') if !_.isObject(services)
-		servicesArray = _.toPairs(services)
-		Promise.each servicesArray, ([ serviceName, service ]) ->
-			throw new Error("Service #{serviceName} has no image specified.") if !service.image
-			docker.getImage(service.image).inspectAsync()
-			.catch ->
-				dockerUtils.pullAndProtectImage(service.image, reportStatus)
-			.then ->
-				utils.validateKeys(service, utils.validComposeOptions)
-			.then ->
-				services[serviceName].volumes = utils.defaultBinds(composeDataPath(appId, serviceName))
-		.then ->
-			writeComposeFile(composeSpec, composePathDst(appId))
-	.then ->
-		runComposeCommand(['up', '-d'], appId, reportStatus)
-	.catch (err) ->
-		msg = err?.message or err
-		reportStatus(error: msg)
-		throw err
-
-# Runs docker-compose down using the compose YAML at "path".
-# Reports status and errors in JSON to the onStatus function.
-exports.down = (appId, onStatus) ->
-	onStatus ?= console.log.bind(console)
-	reportStatus = (status) ->
-		try onStatus(status)
-	runComposeCommand([ 'down' ], appId, reportStatus)
-	.catch (err) ->
-		msg = err?.message or err
-		reportStatus(error: msg)
-		throw err

src/db.coffee

@@ -56,19 +56,6 @@ knex.init = Promise.all([
 				# When updating from older supervisors, config can be null
 				knex('app').update({ config: '{}' }).whereNull('config')
 
-	knex.schema.hasTable('image')
-	.then (exists) ->
-		if not exists
-			knex.schema.createTable 'image', (t) ->
-				t.increments('id').primary()
-				t.string('repoTag')
-	knex.schema.hasTable('container')
-	.then (exists) ->
-		if not exists
-			knex.schema.createTable 'container', (t) ->
-				t.increments('id').primary()
-				t.string('containerId')
-
 	knex.schema.hasTable('dependentApp')
 	.then (exists) ->
 		if not exists

src/docker-utils.coffee

@@ -142,9 +142,6 @@ do ->
 	exports.cleanupContainersAndImages = (extraImagesToIgnore = []) ->
 		Promise.using writeLockImages(), ->
 			Promise.join(
-				knex('image').select('repoTag')
-				.map ({ repoTag }) ->
-					normalizeRepoTag(repoTag)
 				knex('app').select()
 				.map ({ imageId }) ->
 					normalizeRepoTag(imageId)
@@ -157,7 +154,7 @@ do ->
 					image.NormalizedRepoTags = Promise.map(image.RepoTags, normalizeRepoTag)
 					Promise.props(image)
 				Promise.map(extraImagesToIgnore, normalizeRepoTag)
-				(locallyCreatedTags, apps, dependentApps, supervisorTag, images, normalizedExtraImages) ->
+				(apps, dependentApps, supervisorTag, images, normalizedExtraImages) ->
 					imageTags = _.map(images, 'NormalizedRepoTags')
 					supervisorTags = _.filter imageTags, (tags) ->
 						_.includes(tags, supervisorTag)
@@ -170,11 +167,10 @@ do ->
 					supervisorTags = _.flatten(supervisorTags)
 					appTags = _.flatten(appTags)
 					extraTags = _.flatten(extraTags)
-					locallyCreatedTags = _.flatten(locallyCreatedTags)
 
-					return { images, supervisorTags, appTags, locallyCreatedTags, extraTags }
+					return { images, supervisorTags, appTags, extraTags }
 			)
-			.then ({ images, supervisorTags, appTags, locallyCreatedTags, extraTags }) ->
+			.then ({ images, supervisorTags, appTags, extraTags }) ->
 				# Cleanup containers first, so that they don't block image removal.
 				docker.listContainersAsync(all: true)
 				.filter (containerInfo) ->
@@ -183,8 +179,6 @@ do ->
 					.then (repoTag) ->
 						if _.includes(appTags, repoTag)
 							return false
-						if _.includes(locallyCreatedTags, repoTag)
-							return false
 						if _.includes(extraTags, repoTag)
 							return false
 						if !_.includes(supervisorTags, repoTag)
@@ -198,7 +192,7 @@ do ->
 				.then ->
 					imagesToClean = _.reject images, (image) ->
 						_.some image.NormalizedRepoTags, (tag) ->
-							return _.includes(appTags, tag) or _.includes(supervisorTags, tag) or _.includes(locallyCreatedTags, tag) or _.includes(extraTags, tag)
+							return _.includes(appTags, tag) or _.includes(supervisorTags, tag) or _.includes(extraTags, tag)
 					Promise.map imagesToClean, (image) ->
 						Promise.map image.RepoTags.concat(image.Id), (tag) ->
 							docker.getImage(tag).removeAsync(force: true)
@@ -222,207 +216,6 @@ do ->
 			repoTag += ':latest'
 		return repoTag
 
-	sanitizeQuery = (query) ->
-		_.omit(query, 'apikey')
-
-	exports.createImage = (req, res) ->
-		{ registry, repo, tag, fromImage } = req.query
-		if fromImage?
-			repoTag = buildRepoTag(fromImage, tag)
-		else
-			repoTag = buildRepoTag(repo, tag, registry)
-		Promise.using writeLockImages(), ->
-			knex('image').select().where({ repoTag })
-			.then ([ img ]) ->
-				knex('image').insert({ repoTag }) if !img?
-			.then ->
-				if fromImage?
-					docker.createImageAsync({ fromImage, tag })
-				else
-					docker.importImageAsync(req, { repo, tag, registry })
-			.then (stream) ->
-				new Promise (resolve, reject) ->
-					stream.on('error', reject)
-					.on('response', -> resolve())
-					.pipe(res)
-		.catch (err) ->
-			res.status(500).send(err?.message or err or 'Unknown error')
-
-	exports.pullAndProtectImage = (image, onProgress) ->
-		repoTag = buildRepoTag(image)
-		Promise.using writeLockImages(), ->
-			knex('image').select().where({ repoTag })
-			.then ([ img ]) ->
-				knex('image').insert({ repoTag }) if !img?
-			.then ->
-				dockerProgress.pull(repoTag, onProgress)
-
-	exports.getImageTarStream = (image) ->
-		docker.getImage(image).getAsync()
-
-	exports.loadImage = (req, res) ->
-		Promise.using writeLockImages(), ->
-			docker.listImagesAsync()
-			.then (oldImages) ->
-				docker.loadImageAsync(req)
-				.then ->
-					docker.listImagesAsync()
-				.then (newImages) ->
-					oldTags = _.flatten(_.map(oldImages, 'RepoTags'))
-					newTags = _.flatten(_.map(newImages, 'RepoTags'))
-					createdTags = _.difference(newTags, oldTags)
-					Promise.map createdTags, (repoTag) ->
-						knex('image').insert({ repoTag })
-			.then ->
-				res.sendStatus(200)
-		.catch (err) ->
-			res.status(500).send(err?.message or err or 'Unknown error')
-
-	exports.deleteImage = (req, res) ->
-		imageName = req.params[0]
-		Promise.using writeLockImages(), ->
-			knex('image').select().where('repoTag', imageName)
-			.then (images) ->
-				throw new Error('Only images created via the Supervisor can be deleted.') if images.length == 0
-				knex('image').where('repoTag', imageName).delete()
-			.then ->
-				docker.getImage(imageName).removeAsync(sanitizeQuery(req.query))
-				.then (data) ->
-					res.json(data)
-		.catch (err) ->
-			res.status(500).send(err?.message or err or 'Unknown error')
-
-	exports.listImages = (req, res) ->
-		docker.listImagesAsync(sanitizeQuery(req.query))
-		.then (images) ->
-			res.json(images)
-		.catch (err) ->
-			res.status(500).send(err?.message or err or 'Unknown error')
-
-	docker.modem.dialAsync = Promise.promisify(docker.modem.dial)
-	createContainer = (options, internalId) ->
-		Promise.using writeLockImages(), ->
-			Promise.join(
-				knex('image').select().where('repoTag', options.Image)
-				device.isResinOSv1()
-				(images, isV1) ->
-					throw new Error('Only images created via the Supervisor can be used for creating containers.') if images.length == 0
-					knex.transaction (tx) ->
-						Promise.try ->
-							return internalId if internalId?
-							tx.insert({}, 'id').into('container')
-							.then ([ id ]) ->
-								return id
-						.then (id) ->
-							options.HostConfig ?= {}
-							options.Volumes ?= {}
-							_.assign(options.Volumes, utils.defaultVolumes(isV1))
-							options.HostConfig.Binds = utils.defaultBinds("containers/#{id}", isV1)
-							query = ''
-							query = "name=#{options.Name}&" if options.Name?
-							optsf =
-								path: "/containers/create?#{query}"
-								method: 'POST'
-								options: options
-								statusCodes:
-									200: true
-									201: true
-									404: 'no such container'
-									406: 'impossible to attach'
-									500: 'server error'
-							utils.validateKeys(options, utils.validContainerOptions)
-							.then ->
-								utils.validateKeys(options.HostConfig, utils.validHostConfigOptions)
-							.then ->
-								docker.modem.dialAsync(optsf)
-							.then (data) ->
-								containerId = data.Id
-								tx('container').update({ containerId }).where({ id })
-								.return(data)
-			)
-	exports.createContainer = (req, res) ->
-		createContainer(req.body)
-		.then (data) ->
-			res.json(data)
-		.catch (err) ->
-			res.status(500).send(err?.message or err or 'Unknown error')
-
-	startContainer = (containerId, options) ->
-		utils.validateKeys(options, utils.validHostConfigOptions)
-		.then ->
-			docker.getContainer(containerId).startAsync(options)
-	exports.startContainer = (req, res) ->
-		startContainer(req.params.id, req.body)
-		.then (data) ->
-			res.json(data)
-		.catch (err) ->
-			res.status(500).send(err?.message or err or 'Unknown error')
-
-	stopContainer = (containerId, options) ->
-		container = docker.getContainer(containerId)
-		knex('app').select()
-		.then (apps) ->
-			throw new Error('Cannot stop an app container') if _.some(apps, { containerId })
-			container.inspectAsync()
-		.then (cont) ->
-			throw new Error('Cannot stop supervisor container') if cont.Name == '/resin_supervisor' or _.some(cont.Names, (n) -> n == '/resin_supervisor')
-			container.stopAsync(options)
-	exports.stopContainer = (req, res) ->
-		stopContainer(req.params.id, sanitizeQuery(req.query))
-		.then (data) ->
-			res.json(data)
-		.catch (err) ->
-			res.status(500).send(err?.message or err or 'Unknown error')
-
-	deleteContainer = (containerId, options) ->
-		container = docker.getContainer(containerId)
-		knex('app').select()
-		.then (apps) ->
-			throw new Error('Cannot remove an app container') if _.some(apps, { containerId })
-			container.inspectAsync()
-		.then (cont) ->
-			throw new Error('Cannot remove supervisor container') if cont.Name == '/resin_supervisor' or _.some(cont.Names, (n) -> n == '/resin_supervisor')
-			if options.purge
-				knex('container').select().where({ containerId })
-				.then (contFromDB) ->
-					# This will also be affected by #115. Should fix when we fix that.
-					rimraf(utils.getDataPath("containers/#{contFromDB.id}"))
-				.then ->
-					knex('container').where({ containerId }).del()
-		.then ->
-			container.removeAsync(options)
-	exports.deleteContainer = (req, res) ->
-		deleteContainer(req.params.id, sanitizeQuery(req.query))
-		.then (data) ->
-			res.json(data)
-		.catch (err) ->
-			res.status(500).send(err?.message or err or 'Unknown error')
-
-	exports.listContainers = (req, res) ->
-		docker.listContainersAsync(sanitizeQuery(req.query))
-		.then (containers) ->
-			res.json(containers)
-		.catch (err) ->
-			res.status(500).send(err?.message or err or 'Unknown error')
-
-	exports.updateContainer = (req, res) ->
-		{ oldContainerId } = req.query
-		return res.status(400).send('Missing oldContainerId') if !oldContainerId?
-		knex('container').select().where({ containerId: oldContainerId })
-		.then ([ oldContainer ]) ->
-			return res.status(404).send('Old container not found') if !oldContainer?
-			stopContainer(oldContainerId, t: 10)
-			.then ->
-				deleteContainer(oldContainerId, v: true)
-			.then ->
-				createContainer(req.body, oldContainer.id)
-			.tap (data) ->
-				startContainer(data.Id)
-		.then (data) ->
-			res.json(data)
-		.catch (err) ->
-			res.status(500).send(err?.message or err or 'Unknown error')
-
 	exports.getImageEnv = (id) ->
 		docker.getImage(id).inspectAsync()
 		.get('Config').get('Env')