Various improvements and fixes to how compositions are handled

Change the way we get the network gateway to set up the supervisor API address.

Added support for cap_add, cap_drop and devices.

Some fixes like missing fat arrows and removing leftover code.

Signed-off-by: Pablo Carranza Velez <pablo@resin.io>

src/application-manager.coffee

@@ -597,15 +597,22 @@ module.exports = class ApplicationManager extends EventEmitter
 			return dbApp
 
 	createTargetService: (service, opts) ->
+		NotFoundErr = (err) -> err.statusCode == 404
 		serviceOpts = {
 			serviceName: service.serviceName
 		}
 		_.assign(serviceOpts, opts)
+		Promise.join(
 			@images.inspectByName(service.image)
 			.catchReturn(undefined)
-		.then (imageInfo) ->
+			@docker.getNetworkGateway(service.network_mode ? service.appId)
+			.catchReturn(NotFoundErr, null)
+			.catchReturn(@docker.InvalidNetGatewayError, null)
+			(imageInfo, apiHostForNetwork) ->
 				serviceOpts.imageInfo = imageInfo
+				serviceOpts.supervisorApiHost = apiHostForNetwork if apiHostForNetwork?
 				return new Service(service, serviceOpts)
+		)
 
 	normaliseAndExtendAppFromDB: (app) =>
 		Promise.join(
@@ -619,8 +626,10 @@ module.exports = class ApplicationManager extends EventEmitter
 				}
 				_.assign(configOpts, opts)
 				volumes = JSON.parse(app.volumes)
-				_.forEach volumes, (v) ->
+				volumes = _.mapValues volumes, (volumeConfig) ->
-					v.labels ?= {}
+					volumeConfig ?= {}
+					volumeConfig.labels ?= {}
+					return volumeConfig
 				Promise.map(JSON.parse(app.services), (service) => @createTargetService(service, configOpts))
 				.then (services) ->
 					# If a named volume is defined in a service, we add it app-wide so that we can track it and purge it

src/compose/networks.coffee

@@ -26,7 +26,7 @@ module.exports = class Networks
 		.then (networks) ->
 			_.filter(networks, (v) -> v.appId == appId)
 
-	get: (name) ->
+	get: (name) =>
 		@docker.getNetwork(name).inspect()
 		.then (network) ->
 			return @format(network)
@@ -45,7 +45,7 @@ module.exports = class Networks
 			@logger.logSystemEvent(logTypes.createNetworkError, { network: { name }, error: err })
 			throw err
 
-	remove: ({ name }) ->
+	remove: ({ name }) =>
 		@logger.logSystemEvent(logTypes.removeNetwork, { network: { name } })
 		@docker.getNetwork(name).remove()
 		.catch (err) =>

src/compose/service-manager.coffee

@@ -124,7 +124,7 @@ module.exports = class ServiceManager extends EventEmitter
 					alreadyStarted = true
 					return
 
-				if statusCode is '500' and err.json.trim().match(/exec format error$/)
+				if statusCode is '500' and err.message?.trim?()?.match(/exec format error$/)
 					# Provide a friendlier error message for "exec format error"
 					@config.get('deviceType')
 					.then (deviceType) ->
@@ -132,7 +132,7 @@ module.exports = class ServiceManager extends EventEmitter
 				else
 					# rethrow the same error
 					throw err
-			.catch (err) ->
+			.catch (err) =>
 				# If starting the container failed, we remove it so that it doesn't litter
 				container.remove(v: true)
 				.finally =>
@@ -177,6 +177,8 @@ module.exports = class ServiceManager extends EventEmitter
 	getByDockerContainerId: (containerId) =>
 		@docker.getContainer(containerId).inspect()
 		.then (container) ->
+			if !container.Config.Labels['io.resin.supervised']?
+				return null
 			return Service.fromContainer(container)
 		.catchReturn(null)
 

src/compose/service.coffee

@@ -34,6 +34,13 @@ defaultBinds = (appId, serviceName) ->
 		"#{updateLock.lockPath(appId, serviceName)}:/tmp/resin"
 	]
 
+formatDevices = (devices) ->
+	return _.map devices, (device) ->
+		[ PathOnHost, PathInContainer, CgroupPermissions ] = device.split(':')
+		PathInContainer ?= PathOnHost
+		CgroupPermissions ?= 'rwm'
+		return { PathOnHost, PathInContainer, CgroupPermissions }
+
 module.exports = class Service
 	constructor: (serviceProperties, opts = {}) ->
 		{
@@ -61,6 +68,9 @@ module.exports = class Service
 			@cap_drop
 			@commit
 			@status
+			@devices
+			@exposedPorts
+			@portBindings
 		} = serviceProperties
 		@privileged ?= false
 		@volumes ?= []
@@ -71,6 +81,9 @@ module.exports = class Service
 		@expose ?= []
 		@cap_add ?= []
 		@cap_drop ?= []
+		@devices ?= []
+		@exposedPorts ?= {}
+		@portBindings ?= {}
 		@network_mode ?= @appId.toString()
 		if @releaseId?
 			@releaseId = @releaseId.toString()
@@ -83,7 +96,9 @@ module.exports = class Service
 			@extendEnvVars(opts)
 			@extendLabels(opts.imageInfo)
 			@extendAndSanitiseVolumes(opts.imageInfo)
-
+			@extendAndSanitiseExposedPorts(opts.imageInfo)
+			{ @exposedPorts, @portBindings } = @getPortsAndPortBindings()
+			@devices = formatDevices(@devices)
 			if checkTruthy(@labels['io.resin.features.dbus'])
 				@volumes.push('/run/dbus:/host/run/dbus')
 			if checkTruthy(@labels['io.resin.features.kernel_modules'])
@@ -131,6 +146,17 @@ module.exports = class Service
 		@labels['io.resin.commit'] = @commit
 		return @labels
 
+	extendAndSanitiseExposedPorts: (imageInfo) =>
+		@expose = _.clone(@expose)
+		@expose = _.map(@expose, String)
+		if imageInfo?.Config?.ExposedPorts?
+			_.forEach imageInfo.Config.ExposedPorts, (v, k) =>
+				port = k.match(/^([0-9]*)\/tcp$/)?[1]
+				if port? and !_.find(@expose, port)
+					@expose.push(port)
+
+		return @expose
+
 	extendAndSanitiseVolumes: (imageInfo) =>
 		volumes = []
 		_.forEach @volumes, (vol) ->
@@ -224,23 +250,27 @@ module.exports = class Service
 			containerId: container.Id
 			cap_add: container.HostConfig.CapAdd
 			cap_drop: container.HostConfig.CapDrop
+			devices: container.HostConfig.Devices
 			status
+			exposedPorts: container.Config.ExposedPorts
+			portBindings: container.HostConfig.PortBindings
 		}
 		return new Service(service)
 
 	# TODO: map ports for any of the possible formats "container:host/protocol", port ranges, etc.
 	getPortsAndPortBindings: =>
-		ports = {}
+		exposedPorts = {}
 		portBindings = {}
 		if @ports?
 			_.forEach @ports, (port) ->
-				[ hostPort, containerPort ] = port.split(':')
+				[ hostPort, containerPort ] = port.toString().split(':')
-				ports[containerPort + '/tcp'] = {}
+				containerPort ?= hostPort
-				portBindings[containerPort + '/tcp'] = [ HostPort: hostPort ]
+				exposedPorts[containerPort + '/tcp'] = {}
+				portBindings[containerPort + '/tcp'] = [ { HostIp: '', HostPort: hostPort } ]
 		if @expose?
 			_.forEach @expose, (port) ->
-				ports[port + '/tcp'] = {}
+				exposedPorts[port + '/tcp'] = {}
-		return { ports, portBindings }
+		return { exposedPorts, portBindings }
 
 	getBindsAndVolumes: =>
 		binds = []
@@ -254,7 +284,6 @@ module.exports = class Service
 		return { binds, volumes }
 
 	toContainerConfig: =>
-		{ ports, portBindings } = @getPortsAndPortBindings()
 		{ binds, volumes } = @getBindsAndVolumes()
 
 		conf = {
@@ -265,16 +294,17 @@ module.exports = class Service
 			Tty: true
 			Volumes: volumes
 			Env: _.map @environment, (v, k) -> k + '=' + v
-			ExposedPorts: ports
+			ExposedPorts: @exposedPorts
 			Labels: @labels
 			HostConfig:
 				Privileged: @privileged
 				NetworkMode: @network_mode
-				PortBindings: portBindings
+				PortBindings: @portBindings
 				Binds: binds
 				RestartPolicy: @restartPolicy
 				CapAdd: @cap_add
 				CapDrop: @cap_drop
+				Devices: @devices
 		}
 		# If network mode is the default network for this app, add alias for serviceName
 		if @network_mode == @appId.toString()
@@ -296,13 +326,14 @@ module.exports = class Service
 			'restartPolicy'
 			'labels'
 			'environment'
-			'cap_add'
+			'portBindings'
-			'cap_drop'
+			'exposedPorts'
 		]
 		arraysToCompare = [
-			'ports'
-			'expose'
 			'volumes'
+			'devices'
+			'cap_add'
+			'cap_drop'
 		]
 		return _.isEqual(_.pick(this, propertiesToCompare), _.pick(otherService, propertiesToCompare)) and
 			_.every arraysToCompare, (property) =>

src/compose/volumes.coffee

@@ -101,8 +101,8 @@ module.exports = class Volumes
 			@logger.logSystemEvent(logTypes.removeVolumeError, { volume: { name }, error: err })
 
 	isEqualConfig: (current, target) ->
-		currentOpts = current.driver_opts ? {}
+		currentOpts = current?.driver_opts ? {}
-		targetOpts = target.driver_opts ? {}
+		targetOpts = target?.driver_opts ? {}
-		currentLabels = current.labels ? {}
+		currentLabels = current?.labels ? {}
-		targetLabels = target.labels ? {}
+		targetLabels = target?.labels ? {}
 		return _.isEqual(currentLabels, targetLabels) and _.isEqual(currentOpts, targetOpts)

src/event-tracker.coffee

@@ -36,9 +36,7 @@ module.exports = class EventTracker
 
 		properties = _.cloneDeep(properties)
 
-		# Don't log private env vars (e.g. api keys)
+		# Don't log private env vars (e.g. api keys) or other secrets - use a whitelist to mask what we send
-		if properties?.service?.environment?
-			delete properties.service.environment
 		properties = mask(properties, mixpanelMask)
 		@_logEvent('Event:', ev, JSON.stringify(properties))
 		return if !@_client?

src/lib/docker-utils.coffee

@@ -3,6 +3,7 @@ DockerToolbelt = require 'docker-toolbelt'
 { DockerProgress } = require 'docker-progress'
 Promise = require 'bluebird'
 dockerDelta = require 'docker-delta'
+TypedError = require 'typed-error'
 _ = require 'lodash'
 { request, resumable } = require './request'
 { envArrayToObject } = require './conversions'
@@ -30,6 +31,7 @@ module.exports = class DockerUtils extends DockerToolbelt
 		super(opts)
 		@dockerProgress = new DockerProgress(dockerToolbelt: this)
 		@supervisorTagPromise = @normaliseImageName(constants.supervisorImage)
+		@InvalidNetGatewayError = class InvalidNetGatewayError extends TypedError
 		return this
 
 	getRepoAndTag: (image) =>
@@ -127,9 +129,14 @@ module.exports = class DockerUtils extends DockerToolbelt
 	# Maybe switch to just looking at the docker0 interface?
 	# For now we do a hacky thing using the Subnet property...
 	defaultBridgeGateway: =>
-		@getNetwork('bridge').inspect()
+		@getNetworkGateway('bridge')
-		.then (netInfo) ->
+		.catchReturn(@InvalidNetGatewayError, '172.17.0.1')
+
+	getNetworkGateway: (netName) =>
+		return Promise.resolve('127.0.0.1') if netName == 'host'
+		@getNetwork(netName).inspect()
+		.then (netInfo) =>
 			conf = netInfo?.IPAM?.Config?[0]
 			return conf.Gateway if conf?.Gateway?
 			return conf.Subnet.replace('.0/16', '.1') if _.endsWith(conf?.Subnet, '.0/16')
-			return '172.17.0.1'
+			throw new @InvalidNetGatewayError("Cannot determine network gateway for #{netName}")

src/lib/iptables.coffee

@@ -6,6 +6,11 @@ checkAndAddIptablesRule = (rule) ->
 	.catch ->
 		childProcess.execAsync("iptables -A #{rule}")
 
+checkAndInsertIptablesRule = (rule) ->
+	childProcess.execAsync("iptables -C #{rule}")
+	.catch ->
+		childProcess.execAsync("iptables -I #{rule}")
+
 exports.rejectOnAllInterfacesExcept = (allowedInterfaces, port) ->
 	Promise.each allowedInterfaces, (iface) ->
 		checkAndAddIptablesRule("INPUT -p tcp --dport #{port} -i #{iface} -j ACCEPT")
@@ -14,3 +19,12 @@ exports.rejectOnAllInterfacesExcept = (allowedInterfaces, port) ->
 		.catch ->
 			# On systems without REJECT support, fall back to DROP
 			checkAndAddIptablesRule("INPUT -p tcp --dport #{port} -j DROP")
+
+acceptSubnetRule = (allowedSubnet, port) ->
+	return "INPUT -p tcp --dport #{port} -s #{allowedSubnet} -j ACCEPT"
+
+exports.addAcceptedSubnet = (allowedSubnet, port) ->
+	checkAndInsertIptablesRule(acceptSubnetRule(allowedSubnet, port))
+
+exports.isSubnetAccepted = (allowedSubnet, port) ->
+	childProcess.execAsync("iptables -C #{acceptSubnetRule(allowedSubnet, port)}")

src/logger.coffee

@@ -133,12 +133,9 @@ module.exports = class Logger
 		if obj.error?
 			# Report the message from the original cause to the user.
 			errMessage = obj.error.message
-			if _.isEmpty(errMessage)
-				errMessage = obj.error.json
-			if _.isEmpty(errMessage)
-				errMessage = obj.error.reason
 			if _.isEmpty(errMessage)
 				errMessage = 'Unknown cause'
+				console.log('Warning: invalid error message', obj.error)
 			message += " due to '#{errMessage}'"
 		@logSystemMessage(message, obj, logType.eventName)
 		return