Add balena.yml handling and --draft to balena deploy release creation

This change allows use of a contract and release semver when doing a push,
and is part of the larger feature to use the builder as part of a CI/CD pipeline.

Change-type: minor
Signed-off-by: Paul Jonathan <pj@balena.io>

doc/cli.markdown

@@ -3369,6 +3369,13 @@ Set release tags if the image deployment is successful. Multiple
 arguments may be provided, alternating tag keys and values (see examples).
 Hint: Empty values may be specified with "" (bash, cmd.exe) or '""' (PowerShell).
 
+#### --draft
+
+Deploy the release as a draft. Draft releases are ignored
+by the 'track latest' release policy but can be used through release pinning.
+Draft releases can be marked as final through the API. Releases as created
+as final by default unless this option is given.
+
 #### -e, --emulated
 
 Use QEMU for ARM architecture emulation during the image build

lib/commands/deploy.ts

@@ -59,6 +59,7 @@ interface FlagsDef extends ComposeCliFlags, DockerCliFlags {
 	build: boolean;
 	nologupload: boolean;
 	'release-tag'?: string[];
+	draft: boolean;
 	help: void;
 }
 
@@ -136,6 +137,14 @@ ${dockerignoreHelp}
 			`,
 			multiple: true,
 		}),
+		draft: flags.boolean({
+			description: stripIndent`
+				Deploy the release as a draft. Draft releases are ignored
+				by the 'track latest' release policy but can be used through release pinning.
+				Draft releases can be marked as final through the API. Releases as created
+				as final by default unless this option is given.`,
+			default: false,
+		}),
 		...composeCliFlags,
 		...dockerCliFlags,
 		// NOTE: Not supporting -h for help, because of clash with -h in DockerCliFlags
@@ -213,6 +222,7 @@ ${dockerignoreHelp}
 			shouldPerformBuild: !!options.build,
 			shouldUploadLogs: !options.nologupload,
 			buildEmulated: !!options.emulated,
+			createAsDraft: options.draft,
 			buildOpts,
 		});
 		await applyReleaseTagKeysAndValues(
@@ -236,6 +246,7 @@ ${dockerignoreHelp}
 			shouldUploadLogs: boolean;
 			buildEmulated: boolean;
 			buildOpts: any; // arguments to forward to docker build command
+			createAsDraft: boolean;
 		},
 	) {
 		const _ = await import('lodash');
@@ -367,6 +378,8 @@ ${dockerignoreHelp}
 					`Bearer ${auth}`,
 					apiEndpoint,
 					!opts.shouldUploadLogs,
+					composeOpts.projectPath,
+					opts.createAsDraft,
 				);
 			}
 

lib/utils/compose-types.d.ts

@@ -89,6 +89,9 @@ export interface Release {
 		| 'commit'
 		| 'composition'
 		| 'source'
+		| 'is_final'
+		| 'contract'
+		| 'semver'
 		| 'start_timestamp'
 		| 'end_timestamp'
 	>;

lib/utils/compose.js

@@ -175,6 +175,8 @@ export async function originalTarDirectory(dir, param) {
  * @param {number} userId
  * @param {number} appId
  * @param {import('resin-compose-parse').Composition} composition
+ * @param {boolean} draft
+ * @param {string|undefined} semver
  * @returns {Promise<import('./compose-types').Release>}
  */
 export const createRelease = async function (
@@ -183,6 +185,8 @@ export const createRelease = async function (
 	userId,
 	appId,
 	composition,
+	draft,
+	semver,
 ) {
 	const _ = require('lodash');
 	const crypto = require('crypto');
@@ -197,6 +201,8 @@ export const createRelease = async function (
 		composition,
 		source: 'local',
 		commit: crypto.pseudoRandomBytes(16).toString('hex').toLowerCase(),
+		semver,
+		is_final: !draft,
 	});
 
 	return {
@@ -207,6 +213,9 @@ export const createRelease = async function (
 			'commit',
 			'composition',
 			'source',
+			'is_final',
+			'contract',
+			'semver',
 			'start_timestamp',
 			'end_timestamp',
 		]),

lib/utils/compose_ts.ts

@@ -43,6 +43,7 @@ import type { DeviceInfo } from './device/api';
 import { getBalenaSdk, getChalk, stripIndent } from './lazy';
 import Logger = require('./logger');
 import { exists } from './which';
+import jsyaml = require('js-yaml');
 
 /**
  * Given an array representing the raw `--release-tag` flag of the deploy and
@@ -1288,6 +1289,9 @@ async function pushServiceImages(
 	);
 }
 
+// TODO: This should be shared between the CLI & the Builder
+const PLAIN_SEMVER_REGEX = /^([0-9]+)\.([0-9]+)\.([0-9]+)$/;
+
 export async function deployProject(
 	docker: import('dockerode'),
 	logger: Logger,
@@ -1298,6 +1302,8 @@ export async function deployProject(
 	auth: string,
 	apiEndpoint: string,
 	skipLogUpload: boolean,
+	projectPath: string,
+	isDraft: boolean,
 ): Promise<import('balena-release/build/models').ReleaseModel> {
 	const releaseMod = await import('balena-release');
 	const { createRelease, tagServiceImages } = await import('./compose');
@@ -1306,11 +1312,28 @@ export async function deployProject(
 	const prefix = getChalk().cyan('[Info]') + '    ';
 	const spinner = createSpinner();
 
+	const contract = await getContractContent(`${projectPath}/balena.yml`);
+
+	if (contract?.version && !PLAIN_SEMVER_REGEX.test(contract?.version)) {
+		throw new ExpectedError(
+			stripIndent`Error: expected the version field in ${projectPath}/balena.yml to be a valid semver (e.g.: 1.0.0). Got '${contract.version}' instead`,
+		);
+	}
+
 	const $release = await runSpinner(
 		tty,
 		spinner,
 		`${prefix}Creating release...`,
-		() => createRelease(apiEndpoint, auth, userId, appId, composition),
+		() =>
+			createRelease(
+				apiEndpoint,
+				auth,
+				userId,
+				appId,
+				composition,
+				isDraft,
+				contract?.version,
+			),
 	);
 	const { client: pineClient, release, serviceImages } = $release;
 
@@ -1345,6 +1368,11 @@ export async function deployProject(
 	} finally {
 		await runSpinner(tty, spinner, `${prefix}Saving release...`, async () => {
 			release.end_timestamp = new Date();
+
+			// Add contract contents to the release
+			if (contract) {
+				release.contract = JSON.stringify(contract);
+			}
 			if (release.id != null) {
 				await releaseMod.updateRelease(pineClient, release.id, release);
 			}
@@ -1395,6 +1423,35 @@ export function createRunLoop(tick: (...args: any[]) => void) {
 	return runloop;
 }
 
+async function getContractContent(filePath: string): Promise<any | undefined> {
+	let fileContentAsString;
+	try {
+		fileContentAsString = await fs.readFile(filePath, 'utf8');
+	} catch {
+		// File does not exist. Return undefined
+		return;
+	}
+
+	let asJson;
+	try {
+		asJson = jsyaml.load(fileContentAsString) as any;
+	} catch (err) {
+		throw new ExpectedError(
+			`Error parsing file "${filePath}":\n ${err.message}`,
+		);
+	}
+
+	const allowedContractTypes = ['sw.application', 'sw.block'];
+	if (!asJson?.type || !allowedContractTypes.includes(asJson.type)) {
+		throw new ExpectedError(
+			stripIndent`Error: application contract in '${filePath}' needs to
+				define a top level "type" field with an allowed application type.
+				Allowed application types are: ${allowedContractTypes.join(', ')}`,
+		);
+	}
+	return asJson;
+}
+
 function createLogStream(input: Readable) {
 	const split = require('split') as typeof import('split');
 	const stripAnsi = require('strip-ansi-stream');

npm-shrinkwrap.json

@@ -3969,9 +3969,9 @@
       }
     },
     "balena-release": {
-      "version": "3.0.1",
-      "resolved": "https://registry.npmjs.org/balena-release/-/balena-release-3.0.1.tgz",
-      "integrity": "sha512-xzHwTe9zp7Dw4JLPoEGzTseAG1oQNDCAfbEXT7QrzyfXedlqKx/XyP+HoR7Q2ykIVMNUOBm3+7y2/ThMp2sEMw==",
+      "version": "3.1.0",
+      "resolved": "https://registry.npmjs.org/balena-release/-/balena-release-3.1.0.tgz",
+      "integrity": "sha512-FEbu6mdRUXgqZM0UTEz80zS5fLJDPPJ1ztF0+kmFR4VQ1Cr/s1Xn3m35GCREZlq5quOWLQnWr5Xe2TESH7IdIA==",
       "requires": {
         "@types/bluebird": "^3.5.18",
         "@types/node": "^8.0.55",

package.json

@@ -206,7 +206,7 @@
     "balena-image-fs": "^7.0.6",
     "balena-image-manager": "^7.0.3",
     "balena-preload": "^10.5.0",
-    "balena-release": "^3.0.1",
+    "balena-release": "^3.1.0",
     "balena-sdk": "^15.48.0",
     "balena-semver": "^2.3.0",
     "balena-settings-client": "^4.0.7",

tests/commands/deploy.spec.ts

@@ -18,6 +18,7 @@
 import { expect } from 'chai';
 import { promises as fs } from 'fs';
 import * as _ from 'lodash';
+import * as nock from 'nock';
 import * as path from 'path';
 import * as sinon from 'sinon';
 
@@ -79,7 +80,6 @@ describe('balena deploy', function () {
 		api.expectGetMixpanel({ optional: true });
 		api.expectGetConfigDeviceTypes();
 		api.expectGetApplication();
-		api.expectPostRelease();
 		api.expectGetRelease();
 		api.expectGetUser();
 		api.expectGetService({ serviceName: 'main' });
@@ -137,6 +137,7 @@ describe('balena deploy', function () {
 			);
 		}
 
+		api.expectPostRelease({});
 		api.expectPatchImage({});
 		api.expectPatchRelease({});
 		api.expectPostImageLabel();
@@ -154,6 +155,114 @@ describe('balena deploy', function () {
 		});
 	});
 
+	it('should handle the contract and final status for a final (non-draft) release', async () => {
+		const projectPath = path.join(
+			projectsPath,
+			'no-docker-compose',
+			'with-contract',
+		);
+		const expectedFiles: ExpectedTarStreamFiles = {
+			'src/.dockerignore': { fileSize: 16, type: 'file' },
+			'src/start.sh': { fileSize: 30, type: 'file' },
+			Dockerfile: { fileSize: 88, type: 'file' },
+			'balena.yml': { fileSize: 58, type: 'file' },
+		};
+		const responseFilename = 'build-POST.json';
+		const responseBody = await fs.readFile(
+			path.join(dockerResponsePath, responseFilename),
+			'utf8',
+		);
+		const expectedResponseLines = [
+			...commonResponseLines[responseFilename],
+			`[Info] No "docker-compose.yml" file found at "${projectPath}"`,
+			`[Info] Creating default composition with source: "${projectPath}"`,
+			...getDockerignoreWarn1(
+				[path.join(projectPath, 'src', '.dockerignore')],
+				'deploy',
+			),
+		];
+
+		api.expectPostRelease({
+			inspectRequest: (_uri: string, requestBody: nock.Body) => {
+				const body = requestBody.valueOf() as {
+					semver: string;
+					is_final: boolean;
+				};
+				expect(body.semver).to.be.equal('1.5.2');
+				expect(body.is_final).to.be.true;
+			},
+		});
+		api.expectPatchImage({});
+		api.expectPatchRelease({});
+		api.expectPostImageLabel();
+
+		await testDockerBuildStream({
+			commandLine: `deploy testApp --build --source ${projectPath}`,
+			dockerMock: docker,
+			expectedFilesByService: { main: expectedFiles },
+			expectedQueryParamsByService: { main: commonQueryParams },
+			expectedResponseLines,
+			projectPath,
+			responseBody,
+			responseCode: 200,
+			services: ['main'],
+		});
+	});
+
+	it('should handle the contract and final status for a draft release', async () => {
+		const projectPath = path.join(
+			projectsPath,
+			'no-docker-compose',
+			'with-contract',
+		);
+		const expectedFiles: ExpectedTarStreamFiles = {
+			'src/.dockerignore': { fileSize: 16, type: 'file' },
+			'src/start.sh': { fileSize: 30, type: 'file' },
+			Dockerfile: { fileSize: 88, type: 'file' },
+			'balena.yml': { fileSize: 58, type: 'file' },
+		};
+		const responseFilename = 'build-POST.json';
+		const responseBody = await fs.readFile(
+			path.join(dockerResponsePath, responseFilename),
+			'utf8',
+		);
+		const expectedResponseLines = [
+			...commonResponseLines[responseFilename],
+			`[Info] No "docker-compose.yml" file found at "${projectPath}"`,
+			`[Info] Creating default composition with source: "${projectPath}"`,
+			...getDockerignoreWarn1(
+				[path.join(projectPath, 'src', '.dockerignore')],
+				'deploy',
+			),
+		];
+
+		api.expectPostRelease({
+			inspectRequest: (_uri: string, requestBody: nock.Body) => {
+				const body = requestBody.valueOf() as {
+					semver: string;
+					is_final: boolean;
+				};
+				expect(body.semver).to.be.equal('1.5.2');
+				expect(body.is_final).to.be.false;
+			},
+		});
+		api.expectPatchImage({});
+		api.expectPatchRelease({});
+		api.expectPostImageLabel();
+
+		await testDockerBuildStream({
+			commandLine: `deploy testApp --build --draft --source ${projectPath}`,
+			dockerMock: docker,
+			expectedFilesByService: { main: expectedFiles },
+			expectedQueryParamsByService: { main: commonQueryParams },
+			expectedResponseLines,
+			projectPath,
+			responseBody,
+			responseCode: 200,
+			services: ['main'],
+		});
+	});
+
 	it('should update a release with status="failed" on error (single container)', async () => {
 		let sentryStatus: boolean | undefined;
 		const projectPath = path.join(projectsPath, 'no-docker-compose', 'basic');
@@ -176,6 +285,8 @@ describe('balena deploy', function () {
 		// causes the CLI to call process.exit() with process.exitCode = 1
 		const expectedExitCode = 1;
 
+		api.expectPostRelease({});
+
 		// Mock this patch HTTP request to return status code 500, in which case
 		// the release status should be saved as "failed" rather than "success"
 		api.expectPatchImage({
@@ -302,7 +413,7 @@ describe('balena deploy', function () {
 			);
 		}
 
-		// docker.expectGetImages();
+		api.expectPostRelease({});
 		api.expectPatchImage({});
 		api.expectPatchRelease({});
 

tests/nock/balena-api-mock.ts

@@ -122,11 +122,18 @@ export class BalenaAPIMock extends NockMock {
 	/**
 	 * Mocks balena-release call
 	 */
-	public expectPostRelease(opts: ScopeOpts = {}) {
-		this.optPost(/^\/v6\/release($|[(?])/, opts).replyWithFile(
-			200,
-			path.join(apiResponsePath, 'release-POST-v6.json'),
-			jHeader,
+	public expectPostRelease({
+		statusCode = 200,
+		inspectRequest = this.inspectNoOp,
+		optional = false,
+		persist = false,
+	}) {
+		this.optPost(/^\/v6\/release($|[(?])/, { optional, persist }).reply(
+			statusCode,
+			this.getInspectedReplyFileFunction(
+				inspectRequest,
+				path.join(apiResponsePath, 'release-POST-v6.json'),
+			),
 		);
 	}
 

tests/nock/nock-mock.ts

@@ -16,6 +16,7 @@
  */
 
 import * as nock from 'nock';
+import * as fs from 'fs';
 
 export interface ScopeOpts {
 	optional?: boolean;
@@ -103,6 +104,27 @@ export class NockMock {
 		};
 	}
 
+	protected getInspectedReplyFileFunction(
+		inspectRequest: (uri: string, requestBody: nock.Body) => void,
+		replyBodyFile: string,
+	) {
+		return function (
+			this: nock.ReplyFnContext,
+			uri: string,
+			requestBody: nock.Body,
+			cb: (err: NodeJS.ErrnoException | null, result: nock.ReplyBody) => void,
+		) {
+			try {
+				inspectRequest(uri, requestBody);
+			} catch (err) {
+				cb(err, '');
+			}
+
+			const replyBody = fs.readFileSync(replyBodyFile);
+			cb(null, replyBody);
+		};
+	}
+
 	public done() {
 		try {
 			// scope.done() will throw an error if there are expected api calls that have not happened.

tests/test-data/projects/no-docker-compose/with-contract/Dockerfile

@@ -0,0 +1,4 @@
+FROM busybox
+COPY ./src /usr/src/
+RUN chmod a+x /usr/src/*.sh
+CMD ["/usr/src/start.sh"]

tests/test-data/projects/no-docker-compose/with-contract/balena.yml

@@ -0,0 +1,3 @@
+name: testymctestface
+type: sw.application
+version: 1.5.2

tests/test-data/projects/no-docker-compose/with-contract/src/.dockerignore

@@ -0,0 +1 @@
+windows-crlf.sh

tests/test-data/projects/no-docker-compose/with-contract/src/start.sh

@@ -0,0 +1,2 @@
+#!/bin/sh
+echo "Hello, test!"