From 7d568a928b4297671e3776b72f64a6e2845d5f72 Mon Sep 17 00:00:00 2001 From: toochevere Date: Wed, 19 May 2021 22:12:49 +0000 Subject: [PATCH] Add balena.yml handling and `--draft` to `balena deploy` release creation This change allows use of a contract and release semver when doing a push, and is part of the larger feature to use the builder as part of a CI/CD pipeline. Change-type: minor Signed-off-by: Paul Jonathan --- doc/cli.markdown | 7 ++ lib/commands/deploy.ts | 13 ++ lib/utils/compose-types.d.ts | 3 + lib/utils/compose.js | 9 ++ lib/utils/compose_ts.ts | 59 ++++++++- npm-shrinkwrap.json | 6 +- package.json | 2 +- tests/commands/deploy.spec.ts | 115 +++++++++++++++++- tests/nock/balena-api-mock.ts | 17 ++- tests/nock/nock-mock.ts | 22 ++++ .../with-contract/Dockerfile | 4 + .../with-contract/balena.yml | 3 + .../with-contract/src/.dockerignore | 1 + .../with-contract/src/start.sh | 2 + 14 files changed, 251 insertions(+), 12 deletions(-) create mode 100644 tests/test-data/projects/no-docker-compose/with-contract/Dockerfile create mode 100644 tests/test-data/projects/no-docker-compose/with-contract/balena.yml create mode 100644 tests/test-data/projects/no-docker-compose/with-contract/src/.dockerignore create mode 100755 tests/test-data/projects/no-docker-compose/with-contract/src/start.sh diff --git a/doc/cli.markdown b/doc/cli.markdown index 0abf56b3..925bf684 100644 --- a/doc/cli.markdown +++ b/doc/cli.markdown @@ -3369,6 +3369,13 @@ Set release tags if the image deployment is successful. Multiple arguments may be provided, alternating tag keys and values (see examples). Hint: Empty values may be specified with "" (bash, cmd.exe) or '""' (PowerShell). +#### --draft + +Deploy the release as a draft. Draft releases are ignored +by the 'track latest' release policy but can be used through release pinning. +Draft releases can be marked as final through the API. Releases as created +as final by default unless this option is given. + #### -e, --emulated Use QEMU for ARM architecture emulation during the image build diff --git a/lib/commands/deploy.ts b/lib/commands/deploy.ts index 1cea8f21..034b14f3 100644 --- a/lib/commands/deploy.ts +++ b/lib/commands/deploy.ts @@ -59,6 +59,7 @@ interface FlagsDef extends ComposeCliFlags, DockerCliFlags { build: boolean; nologupload: boolean; 'release-tag'?: string[]; + draft: boolean; help: void; } @@ -136,6 +137,14 @@ ${dockerignoreHelp} `, multiple: true, }), + draft: flags.boolean({ + description: stripIndent` + Deploy the release as a draft. Draft releases are ignored + by the 'track latest' release policy but can be used through release pinning. + Draft releases can be marked as final through the API. Releases as created + as final by default unless this option is given.`, + default: false, + }), ...composeCliFlags, ...dockerCliFlags, // NOTE: Not supporting -h for help, because of clash with -h in DockerCliFlags @@ -213,6 +222,7 @@ ${dockerignoreHelp} shouldPerformBuild: !!options.build, shouldUploadLogs: !options.nologupload, buildEmulated: !!options.emulated, + createAsDraft: options.draft, buildOpts, }); await applyReleaseTagKeysAndValues( @@ -236,6 +246,7 @@ ${dockerignoreHelp} shouldUploadLogs: boolean; buildEmulated: boolean; buildOpts: any; // arguments to forward to docker build command + createAsDraft: boolean; }, ) { const _ = await import('lodash'); @@ -367,6 +378,8 @@ ${dockerignoreHelp} `Bearer ${auth}`, apiEndpoint, !opts.shouldUploadLogs, + composeOpts.projectPath, + opts.createAsDraft, ); } diff --git a/lib/utils/compose-types.d.ts b/lib/utils/compose-types.d.ts index ac9dc675..aa6425ed 100644 --- a/lib/utils/compose-types.d.ts +++ b/lib/utils/compose-types.d.ts @@ -89,6 +89,9 @@ export interface Release { | 'commit' | 'composition' | 'source' + | 'is_final' + | 'contract' + | 'semver' | 'start_timestamp' | 'end_timestamp' >; diff --git a/lib/utils/compose.js b/lib/utils/compose.js index e06108ac..1b8794c6 100644 --- a/lib/utils/compose.js +++ b/lib/utils/compose.js @@ -175,6 +175,8 @@ export async function originalTarDirectory(dir, param) { * @param {number} userId * @param {number} appId * @param {import('resin-compose-parse').Composition} composition + * @param {boolean} draft + * @param {string|undefined} semver * @returns {Promise} */ export const createRelease = async function ( @@ -183,6 +185,8 @@ export const createRelease = async function ( userId, appId, composition, + draft, + semver, ) { const _ = require('lodash'); const crypto = require('crypto'); @@ -197,6 +201,8 @@ export const createRelease = async function ( composition, source: 'local', commit: crypto.pseudoRandomBytes(16).toString('hex').toLowerCase(), + semver, + is_final: !draft, }); return { @@ -207,6 +213,9 @@ export const createRelease = async function ( 'commit', 'composition', 'source', + 'is_final', + 'contract', + 'semver', 'start_timestamp', 'end_timestamp', ]), diff --git a/lib/utils/compose_ts.ts b/lib/utils/compose_ts.ts index 96801613..5b493d46 100644 --- a/lib/utils/compose_ts.ts +++ b/lib/utils/compose_ts.ts @@ -43,6 +43,7 @@ import type { DeviceInfo } from './device/api'; import { getBalenaSdk, getChalk, stripIndent } from './lazy'; import Logger = require('./logger'); import { exists } from './which'; +import jsyaml = require('js-yaml'); /** * Given an array representing the raw `--release-tag` flag of the deploy and @@ -1288,6 +1289,9 @@ async function pushServiceImages( ); } +// TODO: This should be shared between the CLI & the Builder +const PLAIN_SEMVER_REGEX = /^([0-9]+)\.([0-9]+)\.([0-9]+)$/; + export async function deployProject( docker: import('dockerode'), logger: Logger, @@ -1298,6 +1302,8 @@ export async function deployProject( auth: string, apiEndpoint: string, skipLogUpload: boolean, + projectPath: string, + isDraft: boolean, ): Promise { const releaseMod = await import('balena-release'); const { createRelease, tagServiceImages } = await import('./compose'); @@ -1306,11 +1312,28 @@ export async function deployProject( const prefix = getChalk().cyan('[Info]') + ' '; const spinner = createSpinner(); + const contract = await getContractContent(`${projectPath}/balena.yml`); + + if (contract?.version && !PLAIN_SEMVER_REGEX.test(contract?.version)) { + throw new ExpectedError( + stripIndent`Error: expected the version field in ${projectPath}/balena.yml to be a valid semver (e.g.: 1.0.0). Got '${contract.version}' instead`, + ); + } + const $release = await runSpinner( tty, spinner, `${prefix}Creating release...`, - () => createRelease(apiEndpoint, auth, userId, appId, composition), + () => + createRelease( + apiEndpoint, + auth, + userId, + appId, + composition, + isDraft, + contract?.version, + ), ); const { client: pineClient, release, serviceImages } = $release; @@ -1345,6 +1368,11 @@ export async function deployProject( } finally { await runSpinner(tty, spinner, `${prefix}Saving release...`, async () => { release.end_timestamp = new Date(); + + // Add contract contents to the release + if (contract) { + release.contract = JSON.stringify(contract); + } if (release.id != null) { await releaseMod.updateRelease(pineClient, release.id, release); } @@ -1395,6 +1423,35 @@ export function createRunLoop(tick: (...args: any[]) => void) { return runloop; } +async function getContractContent(filePath: string): Promise { + let fileContentAsString; + try { + fileContentAsString = await fs.readFile(filePath, 'utf8'); + } catch { + // File does not exist. Return undefined + return; + } + + let asJson; + try { + asJson = jsyaml.load(fileContentAsString) as any; + } catch (err) { + throw new ExpectedError( + `Error parsing file "${filePath}":\n ${err.message}`, + ); + } + + const allowedContractTypes = ['sw.application', 'sw.block']; + if (!asJson?.type || !allowedContractTypes.includes(asJson.type)) { + throw new ExpectedError( + stripIndent`Error: application contract in '${filePath}' needs to + define a top level "type" field with an allowed application type. + Allowed application types are: ${allowedContractTypes.join(', ')}`, + ); + } + return asJson; +} + function createLogStream(input: Readable) { const split = require('split') as typeof import('split'); const stripAnsi = require('strip-ansi-stream'); diff --git a/npm-shrinkwrap.json b/npm-shrinkwrap.json index 19bcbb4b..1a0bbc85 100644 --- a/npm-shrinkwrap.json +++ b/npm-shrinkwrap.json @@ -3969,9 +3969,9 @@ } }, "balena-release": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/balena-release/-/balena-release-3.0.1.tgz", - "integrity": "sha512-xzHwTe9zp7Dw4JLPoEGzTseAG1oQNDCAfbEXT7QrzyfXedlqKx/XyP+HoR7Q2ykIVMNUOBm3+7y2/ThMp2sEMw==", + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/balena-release/-/balena-release-3.1.0.tgz", + "integrity": "sha512-FEbu6mdRUXgqZM0UTEz80zS5fLJDPPJ1ztF0+kmFR4VQ1Cr/s1Xn3m35GCREZlq5quOWLQnWr5Xe2TESH7IdIA==", "requires": { "@types/bluebird": "^3.5.18", "@types/node": "^8.0.55", diff --git a/package.json b/package.json index b1d8272b..ed49aade 100644 --- a/package.json +++ b/package.json @@ -206,7 +206,7 @@ "balena-image-fs": "^7.0.6", "balena-image-manager": "^7.0.3", "balena-preload": "^10.5.0", - "balena-release": "^3.0.1", + "balena-release": "^3.1.0", "balena-sdk": "^15.48.0", "balena-semver": "^2.3.0", "balena-settings-client": "^4.0.7", diff --git a/tests/commands/deploy.spec.ts b/tests/commands/deploy.spec.ts index d39e4c48..b52d5c11 100644 --- a/tests/commands/deploy.spec.ts +++ b/tests/commands/deploy.spec.ts @@ -18,6 +18,7 @@ import { expect } from 'chai'; import { promises as fs } from 'fs'; import * as _ from 'lodash'; +import * as nock from 'nock'; import * as path from 'path'; import * as sinon from 'sinon'; @@ -79,7 +80,6 @@ describe('balena deploy', function () { api.expectGetMixpanel({ optional: true }); api.expectGetConfigDeviceTypes(); api.expectGetApplication(); - api.expectPostRelease(); api.expectGetRelease(); api.expectGetUser(); api.expectGetService({ serviceName: 'main' }); @@ -137,6 +137,7 @@ describe('balena deploy', function () { ); } + api.expectPostRelease({}); api.expectPatchImage({}); api.expectPatchRelease({}); api.expectPostImageLabel(); @@ -154,6 +155,114 @@ describe('balena deploy', function () { }); }); + it('should handle the contract and final status for a final (non-draft) release', async () => { + const projectPath = path.join( + projectsPath, + 'no-docker-compose', + 'with-contract', + ); + const expectedFiles: ExpectedTarStreamFiles = { + 'src/.dockerignore': { fileSize: 16, type: 'file' }, + 'src/start.sh': { fileSize: 30, type: 'file' }, + Dockerfile: { fileSize: 88, type: 'file' }, + 'balena.yml': { fileSize: 58, type: 'file' }, + }; + const responseFilename = 'build-POST.json'; + const responseBody = await fs.readFile( + path.join(dockerResponsePath, responseFilename), + 'utf8', + ); + const expectedResponseLines = [ + ...commonResponseLines[responseFilename], + `[Info] No "docker-compose.yml" file found at "${projectPath}"`, + `[Info] Creating default composition with source: "${projectPath}"`, + ...getDockerignoreWarn1( + [path.join(projectPath, 'src', '.dockerignore')], + 'deploy', + ), + ]; + + api.expectPostRelease({ + inspectRequest: (_uri: string, requestBody: nock.Body) => { + const body = requestBody.valueOf() as { + semver: string; + is_final: boolean; + }; + expect(body.semver).to.be.equal('1.5.2'); + expect(body.is_final).to.be.true; + }, + }); + api.expectPatchImage({}); + api.expectPatchRelease({}); + api.expectPostImageLabel(); + + await testDockerBuildStream({ + commandLine: `deploy testApp --build --source ${projectPath}`, + dockerMock: docker, + expectedFilesByService: { main: expectedFiles }, + expectedQueryParamsByService: { main: commonQueryParams }, + expectedResponseLines, + projectPath, + responseBody, + responseCode: 200, + services: ['main'], + }); + }); + + it('should handle the contract and final status for a draft release', async () => { + const projectPath = path.join( + projectsPath, + 'no-docker-compose', + 'with-contract', + ); + const expectedFiles: ExpectedTarStreamFiles = { + 'src/.dockerignore': { fileSize: 16, type: 'file' }, + 'src/start.sh': { fileSize: 30, type: 'file' }, + Dockerfile: { fileSize: 88, type: 'file' }, + 'balena.yml': { fileSize: 58, type: 'file' }, + }; + const responseFilename = 'build-POST.json'; + const responseBody = await fs.readFile( + path.join(dockerResponsePath, responseFilename), + 'utf8', + ); + const expectedResponseLines = [ + ...commonResponseLines[responseFilename], + `[Info] No "docker-compose.yml" file found at "${projectPath}"`, + `[Info] Creating default composition with source: "${projectPath}"`, + ...getDockerignoreWarn1( + [path.join(projectPath, 'src', '.dockerignore')], + 'deploy', + ), + ]; + + api.expectPostRelease({ + inspectRequest: (_uri: string, requestBody: nock.Body) => { + const body = requestBody.valueOf() as { + semver: string; + is_final: boolean; + }; + expect(body.semver).to.be.equal('1.5.2'); + expect(body.is_final).to.be.false; + }, + }); + api.expectPatchImage({}); + api.expectPatchRelease({}); + api.expectPostImageLabel(); + + await testDockerBuildStream({ + commandLine: `deploy testApp --build --draft --source ${projectPath}`, + dockerMock: docker, + expectedFilesByService: { main: expectedFiles }, + expectedQueryParamsByService: { main: commonQueryParams }, + expectedResponseLines, + projectPath, + responseBody, + responseCode: 200, + services: ['main'], + }); + }); + it('should update a release with status="failed" on error (single container)', async () => { let sentryStatus: boolean | undefined; const projectPath = path.join(projectsPath, 'no-docker-compose', 'basic'); @@ -176,6 +285,8 @@ describe('balena deploy', function () { // causes the CLI to call process.exit() with process.exitCode = 1 const expectedExitCode = 1; + api.expectPostRelease({}); + // Mock this patch HTTP request to return status code 500, in which case // the release status should be saved as "failed" rather than "success" api.expectPatchImage({ @@ -302,7 +413,7 @@ describe('balena deploy', function () { ); } - // docker.expectGetImages(); + api.expectPostRelease({}); api.expectPatchImage({}); api.expectPatchRelease({}); diff --git a/tests/nock/balena-api-mock.ts b/tests/nock/balena-api-mock.ts index 1887ffd3..19294f61 100644 --- a/tests/nock/balena-api-mock.ts +++ b/tests/nock/balena-api-mock.ts @@ -122,11 +122,18 @@ export class BalenaAPIMock extends NockMock { /** * Mocks balena-release call */ - public expectPostRelease(opts: ScopeOpts = {}) { - this.optPost(/^\/v6\/release($|[(?])/, opts).replyWithFile( - 200, - path.join(apiResponsePath, 'release-POST-v6.json'), - jHeader, + public expectPostRelease({ + statusCode = 200, + inspectRequest = this.inspectNoOp, + optional = false, + persist = false, + }) { + this.optPost(/^\/v6\/release($|[(?])/, { optional, persist }).reply( + statusCode, + this.getInspectedReplyFileFunction( + inspectRequest, + path.join(apiResponsePath, 'release-POST-v6.json'), + ), ); } diff --git a/tests/nock/nock-mock.ts b/tests/nock/nock-mock.ts index 5acf4375..de3c9789 100644 --- a/tests/nock/nock-mock.ts +++ b/tests/nock/nock-mock.ts @@ -16,6 +16,7 @@ */ import * as nock from 'nock'; +import * as fs from 'fs'; export interface ScopeOpts { optional?: boolean; @@ -103,6 +104,27 @@ export class NockMock { }; } + protected getInspectedReplyFileFunction( + inspectRequest: (uri: string, requestBody: nock.Body) => void, + replyBodyFile: string, + ) { + return function ( + this: nock.ReplyFnContext, + uri: string, + requestBody: nock.Body, + cb: (err: NodeJS.ErrnoException | null, result: nock.ReplyBody) => void, + ) { + try { + inspectRequest(uri, requestBody); + } catch (err) { + cb(err, ''); + } + + const replyBody = fs.readFileSync(replyBodyFile); + cb(null, replyBody); + }; + } + public done() { try { // scope.done() will throw an error if there are expected api calls that have not happened. diff --git a/tests/test-data/projects/no-docker-compose/with-contract/Dockerfile b/tests/test-data/projects/no-docker-compose/with-contract/Dockerfile new file mode 100644 index 00000000..3bb4ed2f --- /dev/null +++ b/tests/test-data/projects/no-docker-compose/with-contract/Dockerfile @@ -0,0 +1,4 @@ +FROM busybox +COPY ./src /usr/src/ +RUN chmod a+x /usr/src/*.sh +CMD ["/usr/src/start.sh"] diff --git a/tests/test-data/projects/no-docker-compose/with-contract/balena.yml b/tests/test-data/projects/no-docker-compose/with-contract/balena.yml new file mode 100644 index 00000000..3155f269 --- /dev/null +++ b/tests/test-data/projects/no-docker-compose/with-contract/balena.yml @@ -0,0 +1,3 @@ +name: testymctestface +type: sw.application +version: 1.5.2 diff --git a/tests/test-data/projects/no-docker-compose/with-contract/src/.dockerignore b/tests/test-data/projects/no-docker-compose/with-contract/src/.dockerignore new file mode 100644 index 00000000..964e3172 --- /dev/null +++ b/tests/test-data/projects/no-docker-compose/with-contract/src/.dockerignore @@ -0,0 +1 @@ +windows-crlf.sh diff --git a/tests/test-data/projects/no-docker-compose/with-contract/src/start.sh b/tests/test-data/projects/no-docker-compose/with-contract/src/start.sh new file mode 100755 index 00000000..6a3d11d6 --- /dev/null +++ b/tests/test-data/projects/no-docker-compose/with-contract/src/start.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo "Hello, test!"