Merge pull request #2274 from balena-io/deploy-release-versioning

Add balena.yml handling to `balena deploy` release creation

CONTRIBUTING.md

@@ -259,3 +259,12 @@ gotchas to bear in mind:
   `node_modules/balena-sdk/node_modules/balena-errors`  
   In the case of subclasses of `TypedError`, a string comparison may be used instead:  
   `error.name === 'BalenaApplicationNotFound'`
+
+## Further debugging notes
+
+* If you need to selectively run specific tests, `it.only` will not work in cases when authorization is required as part of the test cycle.  In order to target specific tests, control execution via `.mocharc.js` instead.  Here is an example of targeting the `deploy` tests.
+
+	replace: `spec: 'tests/**/*.spec.ts',`
+
+	with: `spec: ['tests/auth/*.spec.ts', 'tests/**/deploy.spec.ts'],`
+

doc/cli.markdown

@@ -3369,6 +3369,13 @@ Set release tags if the image deployment is successful. Multiple
 arguments may be provided, alternating tag keys and values (see examples).
 Hint: Empty values may be specified with "" (bash, cmd.exe) or '""' (PowerShell).
 
+#### --draft
+
+Deploy the release as a draft. Draft releases are ignored
+by the 'track latest' release policy but can be used through release pinning.
+Draft releases can be marked as final through the API. Releases are created
+as final by default unless this option is given.
+
 #### -e, --emulated
 
 Use QEMU for ARM architecture emulation during the image build

lib/commands/deploy.ts

@@ -59,6 +59,7 @@ interface FlagsDef extends ComposeCliFlags, DockerCliFlags {
 	build: boolean;
 	nologupload: boolean;
 	'release-tag'?: string[];
+	draft: boolean;
 	help: void;
 }
 
@@ -136,6 +137,14 @@ ${dockerignoreHelp}
 			`,
 			multiple: true,
 		}),
+		draft: flags.boolean({
+			description: stripIndent`
+				Deploy the release as a draft. Draft releases are ignored
+				by the 'track latest' release policy but can be used through release pinning.
+				Draft releases can be marked as final through the API. Releases are created
+				as final by default unless this option is given.`,
+			default: false,
+		}),
 		...composeCliFlags,
 		...dockerCliFlags,
 		// NOTE: Not supporting -h for help, because of clash with -h in DockerCliFlags
@@ -213,6 +222,7 @@ ${dockerignoreHelp}
 			shouldPerformBuild: !!options.build,
 			shouldUploadLogs: !options.nologupload,
 			buildEmulated: !!options.emulated,
+			createAsDraft: options.draft,
 			buildOpts,
 		});
 		await applyReleaseTagKeysAndValues(
@@ -236,6 +246,7 @@ ${dockerignoreHelp}
 			shouldUploadLogs: boolean;
 			buildEmulated: boolean;
 			buildOpts: any; // arguments to forward to docker build command
+			createAsDraft: boolean;
 		},
 	) {
 		const _ = await import('lodash');
@@ -367,6 +378,8 @@ ${dockerignoreHelp}
 					`Bearer ${auth}`,
 					apiEndpoint,
 					!opts.shouldUploadLogs,
+					composeOpts.projectPath,
+					opts.createAsDraft,
 				);
 			}
 

lib/utils/compose-types.d.ts

@@ -89,6 +89,9 @@ export interface Release {
 		| 'commit'
 		| 'composition'
 		| 'source'
+		| 'is_final'
+		| 'contract'
+		| 'semver'
 		| 'start_timestamp'
 		| 'end_timestamp'
 	>;

lib/utils/compose.js

@@ -175,6 +175,9 @@ export async function originalTarDirectory(dir, param) {
  * @param {number} userId
  * @param {number} appId
  * @param {import('resin-compose-parse').Composition} composition
+ * @param {boolean} draft
+ * @param {string|undefined} semver
+ * @param {string|undefined} contract
  * @returns {Promise<import('./compose-types').Release>}
  */
 export const createRelease = async function (
@@ -183,6 +186,9 @@ export const createRelease = async function (
 	userId,
 	appId,
 	composition,
+	draft,
+	semver,
+	contract,
 ) {
 	const _ = require('lodash');
 	const crypto = require('crypto');
@@ -197,6 +203,9 @@ export const createRelease = async function (
 		composition,
 		source: 'local',
 		commit: crypto.pseudoRandomBytes(16).toString('hex').toLowerCase(),
+		semver,
+		is_final: !draft,
+		contract,
 	});
 
 	return {
@@ -207,6 +216,9 @@ export const createRelease = async function (
 			'commit',
 			'composition',
 			'source',
+			'is_final',
+			'contract',
+			'semver',
 			'start_timestamp',
 			'end_timestamp',
 		]),

lib/utils/compose_ts.ts

@@ -43,6 +43,9 @@ import type { DeviceInfo } from './device/api';
 import { getBalenaSdk, getChalk, stripIndent } from './lazy';
 import Logger = require('./logger');
 import { exists } from './which';
+import jsyaml = require('js-yaml');
+
+const allowedContractTypes = ['sw.application', 'sw.block'];
 
 /**
  * Given an array representing the raw `--release-tag` flag of the deploy and
@@ -1288,6 +1291,9 @@ async function pushServiceImages(
 	);
 }
 
+// TODO: This should be shared between the CLI & the Builder
+const PLAIN_SEMVER_REGEX = /^([0-9]+)\.([0-9]+)\.([0-9]+)$/;
+
 export async function deployProject(
 	docker: import('dockerode'),
 	logger: Logger,
@@ -1298,6 +1304,8 @@ export async function deployProject(
 	auth: string,
 	apiEndpoint: string,
 	skipLogUpload: boolean,
+	projectPath: string,
+	isDraft: boolean,
 ): Promise<import('balena-release/build/models').ReleaseModel> {
 	const releaseMod = await import('balena-release');
 	const { createRelease, tagServiceImages } = await import('./compose');
@@ -1306,11 +1314,29 @@ export async function deployProject(
 	const prefix = getChalk().cyan('[Info]') + '    ';
 	const spinner = createSpinner();
 
+	const contractPath = path.join(projectPath, 'balena.yml');
+	const contract = await getContractContent(contractPath);
+	if (contract?.version && !PLAIN_SEMVER_REGEX.test(contract?.version)) {
+		throw new ExpectedError(stripIndent`\
+			Error: expected the version field in "${contractPath}"
+			to be a basic semver in the format '1.2.3'. Got '${contract.version}' instead`);
+	}
+
 	const $release = await runSpinner(
 		tty,
 		spinner,
 		`${prefix}Creating release...`,
-		() => createRelease(apiEndpoint, auth, userId, appId, composition),
+		() =>
+			createRelease(
+				apiEndpoint,
+				auth,
+				userId,
+				appId,
+				composition,
+				isDraft,
+				contract?.version,
+				contract ? JSON.stringify(contract) : undefined,
+			),
 	);
 	const { client: pineClient, release, serviceImages } = $release;
 
@@ -1395,6 +1421,42 @@ export function createRunLoop(tick: (...args: any[]) => void) {
 	return runloop;
 }
 
+async function getContractContent(
+	filePath: string,
+): Promise<Dictionary<any> | undefined> {
+	let fileContentAsString;
+	try {
+		fileContentAsString = await fs.readFile(filePath, 'utf8');
+	} catch (e) {
+		if (e.code === 'ENOENT') {
+			return; // File does not exist
+		}
+		throw e;
+	}
+
+	let asJson;
+	try {
+		asJson = jsyaml.load(fileContentAsString);
+	} catch (err) {
+		throw new ExpectedError(
+			`Error parsing file "${filePath}":\n ${err.message}`,
+		);
+	}
+
+	if (!isContract(asJson)) {
+		throw new ExpectedError(
+			stripIndent`Error: application contract in '${filePath}' needs to
+				define a top level "type" field with an allowed application type.
+				Allowed application types are: ${allowedContractTypes.join(', ')}`,
+		);
+	}
+	return asJson;
+}
+
+function isContract(obj: any): obj is Dictionary<any> {
+	return obj?.type && allowedContractTypes.includes(obj.type);
+}
+
 function createLogStream(input: Readable) {
 	const split = require('split') as typeof import('split');
 	const stripAnsi = require('strip-ansi-stream');

npm-shrinkwrap.json

@@ -3969,9 +3969,9 @@
       }
     },
     "balena-release": {
-      "version": "3.0.1",
+      "version": "3.2.0",
-      "resolved": "https://registry.npmjs.org/balena-release/-/balena-release-3.0.1.tgz",
+      "resolved": "https://registry.npmjs.org/balena-release/-/balena-release-3.2.0.tgz",
-      "integrity": "sha512-xzHwTe9zp7Dw4JLPoEGzTseAG1oQNDCAfbEXT7QrzyfXedlqKx/XyP+HoR7Q2ykIVMNUOBm3+7y2/ThMp2sEMw==",
+      "integrity": "sha512-jwmAjIZCJ5I46/yQNN+dA73RWlre0+jBVmo2QeJl1pK83obTLyifJeWNVf5irzP8KFE7WQzo9ICK1cCpLtygFA==",
       "requires": {
         "@types/bluebird": "^3.5.18",
         "@types/node": "^8.0.55",

package.json

@@ -206,7 +206,7 @@
     "balena-image-fs": "^7.0.6",
     "balena-image-manager": "^7.0.3",
     "balena-preload": "^10.5.0",
-    "balena-release": "^3.0.1",
+    "balena-release": "^3.2.0",
     "balena-sdk": "^15.48.0",
     "balena-semver": "^2.3.0",
     "balena-settings-client": "^4.0.7",

tests/commands/deploy.spec.ts

@@ -15,9 +15,11 @@
  * limitations under the License.
  */
 
+import type { Request as ReleaseRequest } from 'balena-release';
 import { expect } from 'chai';
 import { promises as fs } from 'fs';
 import * as _ from 'lodash';
+import * as nock from 'nock';
 import * as path from 'path';
 import * as sinon from 'sinon';
 
@@ -79,7 +81,6 @@ describe('balena deploy', function () {
 		api.expectGetMixpanel({ optional: true });
 		api.expectGetConfigDeviceTypes();
 		api.expectGetApplication();
-		api.expectPostRelease();
 		api.expectGetRelease();
 		api.expectGetUser();
 		api.expectGetService({ serviceName: 'main' });
@@ -137,6 +138,7 @@ describe('balena deploy', function () {
 			);
 		}
 
+		api.expectPostRelease({});
 		api.expectPatchImage({});
 		api.expectPatchRelease({});
 		api.expectPostImageLabel();
@@ -154,6 +156,103 @@ describe('balena deploy', function () {
 		});
 	});
 
+	it('should handle the contract and final status for a final (non-draft) release', async () => {
+		const projectPath = path.join(
+			projectsPath,
+			'no-docker-compose',
+			'with-contract',
+		);
+		const expectedFiles: ExpectedTarStreamFiles = {
+			'src/start.sh': { fileSize: 30, type: 'file' },
+			Dockerfile: { fileSize: 88, type: 'file' },
+			'balena.yml': { fileSize: 55, type: 'file' },
+		};
+		const responseFilename = 'build-POST.json';
+		const responseBody = await fs.readFile(
+			path.join(dockerResponsePath, responseFilename),
+			'utf8',
+		);
+		const expectedResponseLines = [
+			...commonResponseLines[responseFilename],
+			`[Info] No "docker-compose.yml" file found at "${projectPath}"`,
+			`[Info] Creating default composition with source: "${projectPath}"`,
+		];
+
+		api.expectPostRelease({
+			inspectRequest: (_uri: string, requestBody: nock.Body) => {
+				const body = requestBody.valueOf() as Partial<ReleaseRequest>;
+				expect(body.contract).to.be.equal(
+					'{"name":"testContract","type":"sw.application","version":"1.5.2"}',
+				);
+				expect(body.is_final).to.be.true;
+			},
+		});
+		api.expectPatchImage({});
+		api.expectPatchRelease({});
+		api.expectPostImageLabel();
+
+		await testDockerBuildStream({
+			commandLine: `deploy testApp --build --source ${projectPath}`,
+			dockerMock: docker,
+			expectedFilesByService: { main: expectedFiles },
+			expectedQueryParamsByService: { main: commonQueryParams },
+			expectedResponseLines,
+			projectPath,
+			responseBody,
+			responseCode: 200,
+			services: ['main'],
+		});
+	});
+
+	it('should handle the contract and final status for a draft release', async () => {
+		const projectPath = path.join(
+			projectsPath,
+			'no-docker-compose',
+			'with-contract',
+		);
+		const expectedFiles: ExpectedTarStreamFiles = {
+			'src/start.sh': { fileSize: 30, type: 'file' },
+			Dockerfile: { fileSize: 88, type: 'file' },
+			'balena.yml': { fileSize: 55, type: 'file' },
+		};
+		const responseFilename = 'build-POST.json';
+		const responseBody = await fs.readFile(
+			path.join(dockerResponsePath, responseFilename),
+			'utf8',
+		);
+		const expectedResponseLines = [
+			...commonResponseLines[responseFilename],
+			`[Info] No "docker-compose.yml" file found at "${projectPath}"`,
+			`[Info] Creating default composition with source: "${projectPath}"`,
+		];
+
+		api.expectPostRelease({
+			inspectRequest: (_uri: string, requestBody: nock.Body) => {
+				const body = requestBody.valueOf() as Partial<ReleaseRequest>;
+				expect(body.contract).to.be.equal(
+					'{"name":"testContract","type":"sw.application","version":"1.5.2"}',
+				);
+				expect(body.semver).to.be.equal('1.5.2');
+				expect(body.is_final).to.be.false;
+			},
+		});
+		api.expectPatchImage({});
+		api.expectPatchRelease({});
+		api.expectPostImageLabel();
+
+		await testDockerBuildStream({
+			commandLine: `deploy testApp --build --draft --source ${projectPath}`,
+			dockerMock: docker,
+			expectedFilesByService: { main: expectedFiles },
+			expectedQueryParamsByService: { main: commonQueryParams },
+			expectedResponseLines,
+			projectPath,
+			responseBody,
+			responseCode: 200,
+			services: ['main'],
+		});
+	});
+
 	it('should update a release with status="failed" on error (single container)', async () => {
 		let sentryStatus: boolean | undefined;
 		const projectPath = path.join(projectsPath, 'no-docker-compose', 'basic');
@@ -176,6 +275,8 @@ describe('balena deploy', function () {
 		// causes the CLI to call process.exit() with process.exitCode = 1
 		const expectedExitCode = 1;
 
+		api.expectPostRelease({});
+
 		// Mock this patch HTTP request to return status code 500, in which case
 		// the release status should be saved as "failed" rather than "success"
 		api.expectPatchImage({
@@ -302,7 +403,7 @@ describe('balena deploy', function () {
 			);
 		}
 
-		// docker.expectGetImages();
+		api.expectPostRelease({});
 		api.expectPatchImage({});
 		api.expectPatchRelease({});
 

tests/config-tests.ts

@@ -43,3 +43,7 @@ import { config as chaiCfg } from 'chai';
 chaiCfg.showDiff = true;
 // enable diff comparison of large objects / arrays
 chaiCfg.truncateThreshold = 0;
+// Because mocks are pointed at "production", we need to make sure this is set to prod.
+// Otherwise if the user has BALENARC_BALENA_URL pointing at something else like staging, tests
+// will fail.
+process.env.BALENARC_BALENA_URL = 'balena-cloud.com';

tests/nock/balena-api-mock.ts

@@ -122,11 +122,18 @@ export class BalenaAPIMock extends NockMock {
 	/**
 	 * Mocks balena-release call
 	 */
-	public expectPostRelease(opts: ScopeOpts = {}) {
+	public expectPostRelease({
-		this.optPost(/^\/v6\/release($|[(?])/, opts).replyWithFile(
+		statusCode = 200,
-			200,
+		inspectRequest = this.inspectNoOp,
+		optional = false,
+		persist = false,
+	}) {
+		this.optPost(/^\/v6\/release($|[(?])/, { optional, persist }).reply(
+			statusCode,
+			this.getInspectedReplyFileFunction(
+				inspectRequest,
 				path.join(apiResponsePath, 'release-POST-v6.json'),
-			jHeader,
+			),
 		);
 	}
 

tests/nock/nock-mock.ts

@@ -16,6 +16,7 @@
  */
 
 import * as nock from 'nock';
+import * as fs from 'fs';
 
 export interface ScopeOpts {
 	optional?: boolean;
@@ -103,6 +104,27 @@ export class NockMock {
 		};
 	}
 
+	protected getInspectedReplyFileFunction(
+		inspectRequest: (uri: string, requestBody: nock.Body) => void,
+		replyBodyFile: string,
+	) {
+		return function (
+			this: nock.ReplyFnContext,
+			uri: string,
+			requestBody: nock.Body,
+			cb: (err: NodeJS.ErrnoException | null, result: nock.ReplyBody) => void,
+		) {
+			try {
+				inspectRequest(uri, requestBody);
+			} catch (err) {
+				cb(err, '');
+			}
+
+			const replyBody = fs.readFileSync(replyBodyFile);
+			cb(null, replyBody);
+		};
+	}
+
 	public done() {
 		try {
 			// scope.done() will throw an error if there are expected api calls that have not happened.

tests/test-data/projects/no-docker-compose/with-contract/Dockerfile

@@ -0,0 +1,4 @@
+FROM busybox
+COPY ./src /usr/src/
+RUN chmod a+x /usr/src/*.sh
+CMD ["/usr/src/start.sh"]

tests/test-data/projects/no-docker-compose/with-contract/balena.yml

@@ -0,0 +1,3 @@
+name: testContract
+type: sw.application
+version: 1.5.2

tests/test-data/projects/no-docker-compose/with-contract/src/start.sh

@@ -0,0 +1,2 @@
+#!/bin/sh
+echo "Hello, test!"