Add support for container contracts

These contracts can specify an OS version and supervisor version that
they require. If any of the containers in a release have requirements
that are not met, the release is rejected, and the previous release
continues to run.

Change-type: minor
Closes: #1086
Signed-off-by: Cameron Diver <cameron@balena.io>

package-lock.json

@@ -4511,12 +4511,14 @@
             "balanced-match": {
               "version": "1.0.0",
               "bundled": true,
-              "dev": true
+              "dev": true,
+              "optional": true
             },
             "brace-expansion": {
               "version": "1.1.11",
               "bundled": true,
               "dev": true,
+              "optional": true,
               "requires": {
                 "balanced-match": "^1.0.0",
                 "concat-map": "0.0.1"
@@ -4536,7 +4538,8 @@
             "concat-map": {
               "version": "0.0.1",
               "bundled": true,
-              "dev": true
+              "dev": true,
+              "optional": true
             },
             "console-control-strings": {
               "version": "1.1.0",
@@ -4684,6 +4687,7 @@
               "version": "3.0.4",
               "bundled": true,
               "dev": true,
+              "optional": true,
               "requires": {
                 "brace-expansion": "^1.1.7"
               }

src/api-binder.ts

@@ -16,6 +16,8 @@ import { EventTracker } from './event-tracker';
 
 import * as constants from './lib/constants';
 import {
+	ContractValidationError,
+	ContractViolationError,
 	DuplicateUuidError,
 	ExchangeKeyError,
 	InternalInconsistencyError,
@@ -28,6 +30,7 @@ import { DeviceApplicationState } from './types/state';
 import log from './lib/supervisor-console';
 
 import DeviceState = require('./device-state');
+import Logger from './logger';
 
 const REPORT_SUCCESS_DELAY = 1000;
 const MAX_REPORT_RETRY_DELAY = 60000;
@@ -44,6 +47,7 @@ export interface APIBinderConstructOpts {
 	db: Database;
 	deviceState: DeviceState;
 	eventTracker: EventTracker;
+	logger: Logger;
 }
 
 interface Device {
@@ -74,6 +78,7 @@ export class APIBinder {
 		[key: string]: any;
 	};
 	private eventTracker: EventTracker;
+	private logger: Logger;
 
 	public balenaApi: PinejsClientRequest | null = null;
 	private cachedBalenaApi: PinejsClientRequest | null = null;
@@ -96,10 +101,12 @@ export class APIBinder {
 		config,
 		deviceState,
 		eventTracker,
+		logger,
 	}: APIBinderConstructOpts) {
 		this.config = config;
 		this.deviceState = deviceState;
 		this.eventTracker = eventTracker;
+		this.logger = logger;
 
 		this.router = this.createAPIBinderRouter(this);
 	}
@@ -580,9 +587,25 @@ export class APIBinder {
 				this.deviceState.triggerApplyTarget({ force, isFromApi });
 			}
 		})
-			.tapCatch(err => {
-				log.error(`Failed to get target state for device: ${err}`);
+			.tapCatch(ContractValidationError, ContractViolationError, e => {
+				log.error(`Could not store target state for device: ${e}`);
+				this.logger.logSystemMessage(
+					`Could not move to new release: ${e.message}`,
+					{},
+					'targetStateRejection',
+					false,
+				);
 			})
+			.tapCatch(
+				(e: unknown) =>
+					!(
+						e instanceof ContractValidationError ||
+						e instanceof ContractViolationError
+					),
+				err => {
+					log.error(`Failed to get target state for device: ${err}`);
+				},
+			)
 			.finally(() => {
 				this.lastTargetStateFetch = process.hrtime();
 			});

src/application-manager.coffee

@@ -9,11 +9,12 @@ path = require 'path'
 constants = require './lib/constants'
 { log } = require './lib/supervisor-console'
 
+{ containerContractsFulfilled, validateContract } = require './lib/contracts'
 { DockerUtils: Docker } = require './lib/docker-utils'
 { LocalModeManager } = require './local-mode'
 updateLock = require './lib/update-lock'
 { checkTruthy, checkInt, checkString } = require './lib/validation'
-{ NotFoundError } = require './lib/errors'
+{ ContractViolationError, ContractValidationError, NotFoundError } = require './lib/errors'
 { pathExistsOnHost } = require './lib/fs-utils'
 
 { ApplicationTargetStateWrapper } = require './target-state'
@@ -683,9 +684,31 @@ module.exports = class ApplicationManager extends EventEmitter
 		)
 
 	setTarget: (apps, dependent , source, trx) =>
-		setInTransaction = (trx) =>
+		# We look at the container contracts here, as if we
+		# cannot run the release, we don't want it to be added
+		# to the database, overwriting the current release. This
+		# is because if we just reject the release, but leave it
+		# in the db, if for any reason the current state stops
+		# running, we won't restart it, leaving the device useless
+		contractsFulfilled = _.mapValues apps, (app) ->
+			serviceContracts = {}
+			_.each app.services, (s) ->
+				if s.contract?
+					try
+						validateContract(s)
+					catch e
+						throw new ContractValidationError(s.serviceName, e.message)
+					serviceContracts[s.serviceName] = s.contract
+
+			if !_.isEmpty(serviceContracts)
+				containerContractsFulfilled(serviceContracts)
+			else
+				{ valid: true }
+
+
+		setInTransaction = (filteredApps, trx) =>
 			Promise.try =>
-				appsArray = _.map apps, (app, appId) ->
+				appsArray = _.map filteredApps, (app, appId) ->
 					appClone = _.clone(app)
 					appClone.appId = checkInt(appId)
 					appClone.source = source
@@ -694,17 +717,34 @@ module.exports = class ApplicationManager extends EventEmitter
 				.tap (appsForDB) =>
 					@targetStateWrapper.setTargetApps(appsForDB, trx)
 				.then (appsForDB) ->
-					trx('app').where({ source }).whereNotIn('appId', _.map(appsForDB, 'appId')).del()
+					trx('app').where({ source }).whereNotIn('appId',
+						# Use apps here, rather than filteredApps, to
+						# avoid removing a release from the database
+						# without an application to replace it.
+						# Currently this will only happen if the release
+						# which would replace it fails a contract
+						# validation check
+						_.map(apps, (_, appId) -> checkInt(appId))
+					).del()
 			.then =>
 				@proxyvisor.setTargetInTransaction(dependent, trx)
 
-		Promise.try =>
+		contractViolators = {}
+		Promise.props(contractsFulfilled).then (fulfilledContracts) ->
+			filteredApps = _.cloneDeep(apps)
+			_.each fulfilledContracts, ({ valid, unmetServices }, appId) ->
+				if not valid
+					contractViolators[apps[appId].name] = unmetServices
+					delete filteredApps[appId]
 			if trx?
-				setInTransaction(trx)
+				setInTransaction(filteredApps, trx)
 			else
 				@db.transaction(setInTransaction)
 		.then =>
 			@_targetVolatilePerImageId = {}
+		.finally ->
+			if not _.isEmpty(contractViolators)
+				throw new ContractViolationError(contractViolators)
 
 	setTargetVolatileForService: (imageId, target) =>
 		@_targetVolatilePerImageId[imageId] ?= {}
@@ -933,6 +973,7 @@ module.exports = class ApplicationManager extends EventEmitter
 		conf = { delta, localMode }
 		if conf.localMode
 			cleanupNeeded = false
+
 		@_inferNextSteps(cleanupNeeded, availableImages, downloading, supervisorNetworkReady, currentState, targetState, ignoreImages, conf, containerIds)
 		.then (nextSteps) =>
 			if ignoreImages and _.some(nextSteps, action: 'fetch')

src/compose/service.ts

@@ -113,6 +113,8 @@ export class Service {
 		service.createdAt = appConfig.createdAt;
 		delete appConfig.createdAt;
 
+		delete appConfig.contract;
+
 		// We don't need this value
 		delete appConfig.commit;
 

src/device-state.coffee

@@ -17,7 +17,12 @@ validation = require './lib/validation'
 systemd = require './lib/systemd'
 updateLock = require './lib/update-lock'
 { singleToMulticontainerApp } = require './lib/migration'
-{ ENOENT, EISDIR, NotFoundError, UpdatesLockedError } = require './lib/errors'
+{
+	ENOENT,
+	EISDIR,
+	NotFoundError,
+	UpdatesLockedError
+} = require './lib/errors'
 
 { DeviceConfig } = require './device-config'
 ApplicationManager = require './application-manager'

src/lib/contracts.ts

@@ -1,3 +1,6 @@
+import { isLeft } from 'fp-ts/lib/Either';
+import * as t from 'io-ts';
+import { reporter } from 'io-ts-reporters';
 import * as _ from 'lodash';
 
 import { Blueprint, Contract, ContractObject } from '@balena/contrato';
@@ -7,9 +10,17 @@ import { InternalInconsistencyError } from './errors';
 import * as osRelease from './os-release';
 import supervisorVersion = require('./supervisor-version');
 
+export { ContractObject };
+
+export interface ServiceContracts {
+	[serviceName: string]: ContractObject;
+}
+
 export async function containerContractsFulfilled(
-	containers: ContractObject[],
-): Promise<boolean> {
+	serviceContracts: ServiceContracts,
+): Promise<{ valid: boolean; unmetServices: string[] }> {
+	const containers = _.values(serviceContracts);
+
 	const osContract = new Contract({
 		slug: 'balenaOS',
 		type: 'sw.os',
@@ -54,7 +65,7 @@ export async function containerContractsFulfilled(
 		);
 	}
 	if (solution.length === 0) {
-		return false;
+		return { valid: false, unmetServices: _.keys(serviceContracts) };
 	}
 
 	// Detect how many containers are present in the resulting
@@ -62,5 +73,50 @@ export async function containerContractsFulfilled(
 	const children = solution[0].getChildren({
 		types: new Set(['sw.container']),
 	});
-	return children.length === containers.length;
+
+	if (children.length === containers.length) {
+		return { valid: true, unmetServices: [] };
+	} else {
+		// Work out which service violated the contracts they
+		// provided
+		const unmetServices = _(serviceContracts)
+			.map((contract, serviceName) => {
+				const found = _.find(children, child => {
+					return _.isEqual((child as any).raw, contract);
+				});
+				if (found == null) {
+					return serviceName;
+				}
+				return;
+			})
+			.filter(n => n != null)
+			.value() as string[];
+		return { valid: false, unmetServices };
+	}
+}
+
+const contractObjectValidator = t.type({
+	slug: t.string,
+	requires: t.union([
+		t.null,
+		t.undefined,
+		t.array(
+			t.type({
+				type: t.string,
+				version: t.union([t.null, t.undefined, t.string]),
+			}),
+		),
+	]),
+});
+
+export function validateContract(
+	contract: unknown,
+): contract is ContractObject {
+	const result = contractObjectValidator.decode(contract);
+
+	if (isLeft(result)) {
+		throw new Error(reporter(result).join('\n'));
+	}
+
+	return true;
 }

src/lib/errors.ts

@@ -1,4 +1,4 @@
-import { endsWith, startsWith } from 'lodash';
+import { endsWith, map, startsWith } from 'lodash';
 import TypedError = require('typed-error');
 
 import { checkInt } from './validation';
@@ -68,3 +68,36 @@ export class ImageAuthenticationError extends TypedError {}
  * See LocalModeManager for a usage example.
  */
 export class SupervisorContainerNotFoundError extends TypedError {}
+
+/**
+ * This error is thrown when a container contract does not
+ * match the minimum we expect from it
+ */
+export class ContractValidationError extends TypedError {
+	constructor(serviceName: string, error: string) {
+		super(
+			`The contract for service ${serviceName} failed validation, with error: ${error}`,
+		);
+	}
+}
+
+/**
+ * This error is thrown when one or releases cannot be ran
+ * as one or more of their container have unmet requirements.
+ * It accepts a map of app names to arrays of service names
+ * which have unmet requirements.
+ */
+export class ContractViolationError extends TypedError {
+	constructor(violators: { [appName: string]: string[] }) {
+		const appStrings = map(
+			violators,
+			(svcs, name) =>
+				`${name}: Services with unmet requirements: ${svcs.join(', ')}`,
+		);
+		super(
+			`Some releases were rejected due to having unmet requirements:\n  ${appStrings.join(
+				'\n  ',
+			)}`,
+		);
+	}
+}

src/lib/os-release.ts

@@ -1,4 +1,3 @@
-import * as Bluebird from 'bluebird';
 import * as _ from 'lodash';
 import fs = require('mz/fs');
 

src/supervisor.ts

@@ -51,6 +51,7 @@ export class Supervisor {
 			db: this.db,
 			deviceState: this.deviceState,
 			eventTracker: this.eventTracker,
+			logger: this.logger,
 		});
 
 		// FIXME: rearchitect proxyvisor to avoid this circular dependency

test/24-contracts.ts

@@ -1,156 +1,235 @@
 import { assert, expect } from 'chai';
 
-import * as semver from 'balena-semver';
+import * as semver from 'semver';
 
 import * as constants from '../src/lib/constants';
-import { containerContractsFulfilled } from '../src/lib/contracts';
+import {
+	containerContractsFulfilled,
+	validateContract,
+} from '../src/lib/contracts';
 import * as osRelease from '../src/lib/os-release';
 import supervisorVersion = require('../src/lib/supervisor-version');
 
 describe('Container contracts', () => {
-	// Because the supervisor version will change whenever the
-	// package.json will, we generate values which are above
-	// and below the current value, and use these to reason
-	// about the contract engine results
-	const supervisorVersionGreater = `${semver.major(supervisorVersion)! +
-		1}.0.0`;
-	const supervisorVersionLesser = `${semver.major(supervisorVersion)! - 1}.0.0`;
-
-	before(async () => {
-		// We ensure that the versions we're using for testing
-		// are the same as the time of implementation, otherwise
-		// these tests could fail or succeed when they shouldn't
-		expect(await osRelease.getOSSemver(constants.hostOSVersionPath)).to.equal(
-			'2.0.6',
-		);
-		assert(semver.gt(supervisorVersionGreater, supervisorVersion));
-		assert(semver.lt(supervisorVersionLesser, supervisorVersion));
-	});
-
-	it('Should correctly run containers with no requirements', async () => {
-		expect(
-			await containerContractsFulfilled([
-				{
-					type: 'sw.container',
-					name: 'user-container',
+	describe('Contract validation', () => {
+		it('should correctly validate a contract with no requirements', () => {
+			assert(
+				validateContract({
 					slug: 'user-container',
-				},
-			]),
-		).to.equal(true);
-		expect(
-			await containerContractsFulfilled([
-				{
-					type: 'sw.container',
-					name: 'user-container',
-					slug: 'user-container1',
-				},
-				{
-					type: 'sw.container',
-					name: 'user-container',
-					slug: 'user-container2',
-				},
-			]),
-		).to.equal(true);
-	});
+				}),
+			);
+		});
 
-	it('should correctly run containers whose requirements are satisfied', async () => {
-		expect(
-			await containerContractsFulfilled([
-				{
-					type: 'sw.container',
+		it('should correctly validate a contract with extra fields', () => {
+			assert(
+				validateContract({
+					slug: 'user-container',
 					name: 'user-container',
+					version: '3.0.0',
+				}),
+			);
+		});
+
+		it('should not validate a contract without the minimum required fields', () => {
+			expect(() => {
+				validateContract({});
+			}).to.throw();
+			expect(() => {
+				validateContract({ name: 'test' });
+			}).to.throw();
+			expect(() => {
+				validateContract({ requires: [] });
+			}).to.throw();
+		});
+
+		it('should correctly validate a contract with requirements', () => {
+			assert(
+				validateContract({
 					slug: 'user-container',
 					requires: [
 						{
 							type: 'sw.os',
-							version: '>2.0.0',
+							version: '>3.0.0',
+						},
+						{
+							type: 'sw.supervisor',
 						},
 					],
-				},
-			]),
-		).to.equal(true);
+				}),
+			);
+		});
 
-		expect(
-			await containerContractsFulfilled([
-				{
-					type: 'sw.container',
-					name: 'user-container',
+		it('should not validate a contract with requirements without the minimum required fields', () => {
+			expect(() =>
+				validateContract({
 					slug: 'user-container',
 					requires: [
 						{
-							type: 'sw.supervisor',
-							version: `<${supervisorVersionGreater}`,
+							version: '>3.0.0',
 						},
 					],
-				},
-			]),
-		).to.equal(true);
-		expect(
-			await containerContractsFulfilled([
-				{
-					type: 'sw.container',
-					name: 'user-container',
-					slug: 'user-container',
-					requires: [
-						{
-							type: 'sw.supervisor',
-							version: `>${supervisorVersionLesser}`,
-						},
-					],
-				},
-			]),
-		).to.equal(true);
-		expect(
-			await containerContractsFulfilled([
-				{
-					type: 'sw.container',
-					name: 'user-container',
-					slug: 'user-container',
-					requires: [
-						{
-							type: 'sw.supervisor',
-							version: `>${supervisorVersionLesser}`,
-						},
-						{
-							type: 'sw.os',
-							version: '<3.0.0',
-						},
-					],
-				},
-			]),
-		).to.equal(true);
-		expect(
-			await containerContractsFulfilled([
-				{
-					type: 'sw.container',
-					name: 'user-container1',
-					slug: 'user-container1',
-					requires: [
-						{
-							type: 'sw.supervisor',
-							version: `>${supervisorVersionLesser}`,
-						},
-					],
-				},
-				{
-					type: 'sw.container',
-					name: 'user-container1',
-					slug: 'user-container1',
-					requires: [
-						{
-							type: 'sw.os',
-							version: '<3.0.0',
-						},
-					],
-				},
-			]),
-		).to.equal(true);
+				}),
+			).to.throw();
+		});
 	});
 
-	it('Should refuse to run containers whose requirements are not satisfied', async () => {
-		expect(
-			await containerContractsFulfilled([
-				{
+	describe('Requirement resolution', () => {
+		// Because the supervisor version will change whenever the
+		// package.json will, we generate values which are above
+		// and below the current value, and use these to reason
+		// about the contract engine results
+		const supervisorVersionGreater = `${semver.major(supervisorVersion)! +
+			1}.0.0`;
+		const supervisorVersionLesser = `${semver.major(supervisorVersion)! -
+			1}.0.0`;
+
+		before(async () => {
+			// We ensure that the versions we're using for testing
+			// are the same as the time of implementation, otherwise
+			// these tests could fail or succeed when they shouldn't
+			expect(await osRelease.getOSSemver(constants.hostOSVersionPath)).to.equal(
+				'2.0.6',
+			);
+			assert(semver.gt(supervisorVersionGreater, supervisorVersion));
+			assert(semver.lt(supervisorVersionLesser, supervisorVersion));
+		});
+
+		it('Should correctly run containers with no requirements', async () => {
+			expect(
+				await containerContractsFulfilled({
+					service: {
+						type: 'sw.container',
+						slug: 'user-container',
+					},
+				}),
+			)
+				.to.have.property('valid')
+				.that.equals(true);
+			expect(
+				await containerContractsFulfilled({
+					service: {
+						type: 'sw.container',
+						slug: 'user-container1',
+					},
+					service2: {
+						type: 'sw.container',
+						slug: 'user-container2',
+					},
+				}),
+			)
+				.to.have.property('valid')
+				.that.equals(true);
+		});
+
+		it('should correctly run containers whose requirements are satisfied', async () => {
+			expect(
+				await containerContractsFulfilled({
+					service: {
+						type: 'sw.container',
+						name: 'user-container',
+						slug: 'user-container',
+						requires: [
+							{
+								type: 'sw.os',
+								version: '>2.0.0',
+							},
+						],
+					},
+				}),
+			)
+				.to.have.property('valid')
+				.that.equals(true);
+
+			expect(
+				await containerContractsFulfilled({
+					service: {
+						type: 'sw.container',
+						name: 'user-container',
+						slug: 'user-container',
+						requires: [
+							{
+								type: 'sw.supervisor',
+								version: `<${supervisorVersionGreater}`,
+							},
+						],
+					},
+				}),
+			)
+				.to.have.property('valid')
+				.that.equals(true);
+
+			expect(
+				await containerContractsFulfilled({
+					service: {
+						type: 'sw.container',
+						name: 'user-container',
+						slug: 'user-container',
+						requires: [
+							{
+								type: 'sw.supervisor',
+								version: `>${supervisorVersionLesser}`,
+							},
+						],
+					},
+				}),
+			)
+				.to.have.property('valid')
+				.that.equals(true);
+
+			expect(
+				await containerContractsFulfilled({
+					service: {
+						type: 'sw.container',
+						name: 'user-container',
+						slug: 'user-container',
+						requires: [
+							{
+								type: 'sw.supervisor',
+								version: `>${supervisorVersionLesser}`,
+							},
+							{
+								type: 'sw.os',
+								version: '<3.0.0',
+							},
+						],
+					},
+				}),
+			)
+				.to.have.property('valid')
+				.that.equals(true);
+			expect(
+				await containerContractsFulfilled({
+					service: {
+						type: 'sw.container',
+						name: 'user-container1',
+						slug: 'user-container1',
+						requires: [
+							{
+								type: 'sw.supervisor',
+								version: `>${supervisorVersionLesser}`,
+							},
+						],
+					},
+					service2: {
+						type: 'sw.container',
+						name: 'user-container1',
+						slug: 'user-container1',
+						requires: [
+							{
+								type: 'sw.os',
+								version: '<3.0.0',
+							},
+						],
+					},
+				}),
+			)
+				.to.have.property('valid')
+				.that.equals(true);
+		});
+
+		it('Should refuse to run containers whose requirements are not satisfied', async () => {
+			let fulfilled = await containerContractsFulfilled({
+				service: {
 					type: 'sw.container',
 					name: 'user-container',
 					slug: 'user-container',
@@ -161,11 +240,16 @@ describe('Container contracts', () => {
 						},
 					],
 				},
-			]),
-		).to.equal(false);
-		expect(
-			await containerContractsFulfilled([
-				{
+			});
+			expect(fulfilled)
+				.to.have.property('valid')
+				.that.equals(false);
+			expect(fulfilled)
+				.to.have.property('unmetServices')
+				.that.deep.equals(['service']);
+
+			fulfilled = await containerContractsFulfilled({
+				service2: {
 					type: 'sw.container',
 					name: 'user-container2',
 					slug: 'user-container2',
@@ -180,11 +264,16 @@ describe('Container contracts', () => {
 						},
 					],
 				},
-			]),
-		).to.equal(false);
-		expect(
-			await containerContractsFulfilled([
-				{
+			});
+			expect(fulfilled)
+				.to.have.property('valid')
+				.that.equals(false);
+			expect(fulfilled)
+				.to.have.property('unmetServices')
+				.that.deep.equals(['service2']);
+
+			fulfilled = await containerContractsFulfilled({
+				service: {
 					type: 'sw.container',
 					name: 'user-container1',
 					slug: 'user-container1',
@@ -195,7 +284,7 @@ describe('Container contracts', () => {
 						},
 					],
 				},
-				{
+				service2: {
 					type: 'sw.container',
 					name: 'user-container2',
 					slug: 'user-container2',
@@ -206,7 +295,13 @@ describe('Container contracts', () => {
 						},
 					],
 				},
-			]),
-		).to.equal(false);
+			});
+			expect(fulfilled)
+				.to.have.property('valid')
+				.that.equals(false);
+			expect(fulfilled)
+				.to.have.property('unmetServices')
+				.that.deep.equals(['service2']);
+		});
 	});
 });