deploy: Fix pushing the images for compositions with more than 70 services

Change-type: patch
See: https://balena.zulipchat.com/#narrow/stream/403752-channel.2Fsupport-help/topic/balena.20deploy.20too.20many.20requests
See: https://github.com/balena-io/balena-cli/pull/1057

.github/actions/publish/action.yml

@@ -28,7 +28,7 @@ runs:
   using: 'composite'
   steps:
     - name: Download custom source artifact
-      uses: actions/download-artifact@f44cd7b40bfd40b6aa1cc1b9b5b7bf03d3c67110 # v4.1.0
+      uses: actions/download-artifact@v3
       with:
         name: custom-${{ github.event.pull_request.head.sha || github.event.head_commit.id }}-${{ runner.os }}-${{ runner.arch }}
         path: ${{ runner.temp }}
@@ -127,9 +127,8 @@ runs:
         XCODE_APP_LOADER_TEAM_ID: ${{ inputs.XCODE_APP_LOADER_TEAM_ID }}
 
     - name: Upload artifacts
-      uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4
+      uses: actions/upload-artifact@v3
       with:
-        name: gh-release-${{ github.event.pull_request.head.sha || github.event.head_commit.id }}-${{ strategy.job-index }}
+        name: gh-release-${{ github.event.pull_request.head.sha || github.event.head_commit.id }}
         path: dist
         retention-days: 1
-        if-no-files-found: error

.github/actions/test/action.yml

@@ -58,7 +58,7 @@ runs:
       run: tar --exclude-vcs -acf ${{ runner.temp }}/custom.tgz .
 
     - name: Upload custom artifact
-      uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4
+      uses: actions/upload-artifact@v3
       with:
         name: custom-${{ github.event.pull_request.head.sha || github.event.head_commit.id }}-${{ runner.os }}-${{ runner.arch }}
         path: ${{ runner.temp }}/custom.tgz

.github/workflows/flowzone.yml

@@ -1,4 +1,5 @@
 name: Flowzone
+
 on:
   pull_request:
     types: [opened, synchronize, closed]
@@ -6,6 +7,7 @@ on:
   pull_request_target:
     types: [opened, synchronize, closed]
     branches: [main, master]
+
 jobs:
   flowzone:
     name: Flowzone
@@ -22,5 +24,7 @@ jobs:
     secrets: inherit
     with:
       custom_runs_on: '[["self-hosted","Linux","distro:focal","X64"],["self-hosted","Linux","distro:focal","ARM64"],["macos-12"],["windows-2019"]]'
+      repo_config: true
+      repo_description: "The official balena CLI tool."
       github_prerelease: false
       restrict_custom_actions: false

.versionbot/CHANGELOG.yml

@@ -1,197 +1,3 @@
-- commits:
-    - subject: Normalize v prefixes in the --version parameter of all commands
-      hash: b7b01ecd5314bddae73b7b062f9d034b3661bcef
-      body: ""
-      footer:
-        Change-type: patch
-        change-type: patch
-      author: Thodoris Greasidis
-      nested: []
-  version: 17.4.10
-  title: ""
-  date: 2024-01-02T12:41:38.978Z
-- commits:
-    - subject: Fix publishing artifacts to gh release
-      hash: 1da5a75c1411bdfece2b60f83095082f6ce68ace
-      body: ""
-      footer:
-        Change-type: patch
-        change-type: patch
-      author: Otávio Jacobi
-      nested: []
-  version: 17.4.9
-  title: ""
-  date: 2023-12-19T23:02:29.500Z
-- commits:
-    - subject: Remove repo config from flowzone.yml
-      hash: bfbc71215c376e815e7d86561d87c5b697ba7482
-      body: |
-        This functionality is being deprecated in Flowzone.
-
-        See: https://github.com/product-os/flowzone/pull/833
-      footer:
-        Change-type: patch
-        change-type: patch
-        Signed-off-by: Kyle Harding <kyle@balena.io>
-        signed-off-by: Kyle Harding <kyle@balena.io>
-      author: Kyle Harding
-      nested: []
-  version: 17.4.8
-  title: ""
-  date: 2023-12-19T21:59:06.220Z
-- commits:
-    - subject: "deploy: Add rate-limiting aware retries for failed requests"
-      hash: 4266dc69514c2177399fc605985196a436d75740
-      body: ""
-      footer:
-        Change-type: patch
-        change-type: patch
-      author: Thodoris Greasidis
-      nested: []
-    - subject: Update dependencies
-      hash: 0ba352258482048bbdb840be7ee9958b491f9b6c
-      body: |
-        Update @balena/compose from 3.0.5 to 3.2.0
-
-        Also updates pinejs-client-request to support
-        using the Retry-After header and dockerode
-        to 3.3.5 to be aligned with @balena/compose.
-      footer:
-        Change-type: patch
-        change-type: patch
-      author: Thodoris Greasidis
-      nested:
-        - commits:
-            - subject: 'release/createClient: Allow specifying the "retry" options'
-              hash: b89b42a838ed2c3a7a8319cbd1b2a7c66a8210ef
-              body: ""
-              footer:
-                Change-type: minor
-                change-type: minor
-              author: Thodoris Greasidis
-              nested: []
-          version: balena-compose-3.2.0
-          title: ""
-          date: 2023-12-05T15:26:57.394Z
-        - commits:
-            - subject: Update dockerode to 3.3.5
-              hash: f5fc932f3203df4df66d38363974e62788e468ff
-              body: ""
-              footer:
-                Change-type: patch
-                change-type: patch
-              author: Pagan Gazzard
-              nested: []
-          version: balena-compose-3.1.3
-          title: ""
-          date: 2023-11-29T14:49:55.816Z
-        - commits:
-            - subject: Use the JSONStream typings from @types/jsonstream
-              hash: 155fdcc8e4e7df67d41152b494e1a80493bb0439
-              body: ""
-              footer:
-                Change-type: patch
-                change-type: patch
-              author: Pagan Gazzard
-              nested: []
-          version: balena-compose-3.1.2
-          title: ""
-          date: 2023-11-29T13:33:49.557Z
-        - commits:
-            - subject: Make use of `pipeline` for piping streams together
-              hash: 1d98cd535a20fa67869da242b0ec7ddd713a4c7b
-              body: ""
-              footer:
-                Change-type: patch
-                change-type: patch
-              author: Pagan Gazzard
-              nested: []
-          version: balena-compose-3.1.1
-          title: ""
-          date: 2023-11-27T12:43:23.880Z
-        - commits:
-            - subject: Allow injecting any PinejsClientCore compatible API client
-              hash: e0ab3ef95f8bc51d2e9055a1f822b8d340f0c587
-              body: ""
-              footer:
-                Change-type: minor
-                change-type: minor
-              author: Thodoris Greasidis
-              nested: []
-          version: balena-compose-3.1.0
-          title: ""
-          date: 2023-11-13T16:27:44.317Z
-        - commits:
-            - subject: "NodeResolver: Refactor the recursion to an async-await loop"
-              hash: bde40f4430bc26a058598a64eeeedbb5ab35eb57
-              body: ""
-              footer:
-                Change-type: patch
-                change-type: patch
-              author: Thodoris Greasidis
-              nested: []
-            - subject: Drop bluebird & bluebird-lru-cache in favor of memoizee
-              hash: 82f90b210d73ff866f5d0546e73d8779db85a504
-              body: ""
-              footer:
-                Change-type: patch
-                change-type: patch
-              author: Thodoris Greasidis
-              nested: []
-          version: balena-compose-3.0.7
-          title: ""
-          date: 2023-11-10T16:10:01.859Z
-        - commits:
-            - subject: Fix the remaining linting errors
-              hash: 51b7893bc6156d0fa7a7821cc583032694ccda98
-              body: ""
-              footer:
-                Change-type: patch
-                change-type: patch
-              author: Thodoris Greasidis
-              nested: []
-            - subject: Remove unnecessary regex escaping
-              hash: 96b76abbcf78abd05157d49a5672a2621124bfe5
-              body: ""
-              footer:
-                Change-type: patch
-                change-type: patch
-              author: Thodoris Greasidis
-              nested: []
-            - subject: Replace the {} type with object
-              hash: dcf907ff124a638f591ae8e3fd80157eae1d1837
-              body: ""
-              footer:
-                Change-type: patch
-                change-type: patch
-              author: Thodoris Greasidis
-              nested: []
-            - subject: Update TypeScript to 5.2.2 and @blaena/lint to v7.2.1
-              hash: b583dd7ce8e964bef47f73dee53e08b7c1286532
-              body: ""
-              footer:
-                Change-type: patch
-                change-type: patch
-              author: Thodoris Greasidis
-              nested: []
-          version: balena-compose-3.0.6
-          title: ""
-          date: 2023-11-10T14:08:35.300Z
-  version: 17.4.7
-  title: ""
-  date: 2023-12-19T14:26:26.818Z
-- commits:
-    - subject: Bump oclif core & use default missing flag handler
-      hash: b9722c67963c9b90e94aec7653ee488957ecd690
-      body: ""
-      footer:
-        Change-type: patch
-        change-type: patch
-      author: Otávio Jacobi
-      nested: []
-  version: 17.4.6
-  title: ""
-  date: 2023-12-08T15:55:47.078Z
 - commits:
     - subject: Stop testing dependency deduplication on the custom test runners
       hash: 65ba63d1a8d231851634830be6d48fbf0e085e47

CHANGELOG.md

@@ -4,64 +4,6 @@ All notable changes to this project will be documented in this file
 automatically by Versionist. DO NOT EDIT THIS FILE MANUALLY!
 This project adheres to [Semantic Versioning](http://semver.org/).
 
-## 17.4.10 - 2024-01-02
-
-* Normalize v prefixes in the --version parameter of all commands [Thodoris Greasidis]
-
-## 17.4.9 - 2023-12-19
-
-* Fix publishing artifacts to gh release [Otávio Jacobi]
-
-## 17.4.8 - 2023-12-19
-
-* Remove repo config from flowzone.yml [Kyle Harding]
-
-## 17.4.7 - 2023-12-19
-
-* deploy: Add rate-limiting aware retries for failed requests [Thodoris Greasidis]
-
-<details>
-<summary> Update dependencies [Thodoris Greasidis] </summary>
-
-> ### balena-compose-3.2.0 - 2023-12-05
-> 
-> * release/createClient: Allow specifying the "retry" options [Thodoris Greasidis]
-> 
-> ### balena-compose-3.1.3 - 2023-11-29
-> 
-> * Update dockerode to 3.3.5 [Pagan Gazzard]
-> 
-> ### balena-compose-3.1.2 - 2023-11-29
-> 
-> * Use the JSONStream typings from @types/jsonstream [Pagan Gazzard]
-> 
-> ### balena-compose-3.1.1 - 2023-11-27
-> 
-> * Make use of `pipeline` for piping streams together [Pagan Gazzard]
-> 
-> ### balena-compose-3.1.0 - 2023-11-13
-> 
-> * Allow injecting any PinejsClientCore compatible API client [Thodoris Greasidis]
-> 
-> ### balena-compose-3.0.7 - 2023-11-10
-> 
-> * NodeResolver: Refactor the recursion to an async-await loop [Thodoris Greasidis]
-> * Drop bluebird & bluebird-lru-cache in favor of memoizee [Thodoris Greasidis]
-> 
-> ### balena-compose-3.0.6 - 2023-11-10
-> 
-> * Fix the remaining linting errors [Thodoris Greasidis]
-> * Remove unnecessary regex escaping [Thodoris Greasidis]
-> * Replace the {} type with object [Thodoris Greasidis]
-> * Update TypeScript to 5.2.2 and @blaena/lint to v7.2.1 [Thodoris Greasidis]
-> 
-
-</details>
-
-## 17.4.6 - 2023-12-08
-
-* Bump oclif core & use default missing flag handler [Otávio Jacobi]
-
 ## 17.4.5 - 2023-12-04
 
 * Stop testing dependency deduplication on the custom test runners [Thodoris Greasidis]

INSTALL-ADVANCED.md

@@ -141,8 +141,7 @@ $ npm install balena-cli --global --production --unsafe-perm
 ```
 
 `--unsafe-perm` is needed when `npm install` is executed as the `root` user (e.g. in a Docker
-container) in order to allow npm scripts like `postinstall` to be executed. The `--global` flag is needed so
-the install uses the `npm-shrinkwrap.json` lockfile when [downloading dependencies](https://docs.npmjs.com/cli/v9/configuring-npm/npm-shrinkwrap-json#description).
+container) in order to allow npm scripts like `postinstall` to be executed.
 
 ## Additional Dependencies
 

lib/commands/config/generate.ts

@@ -259,8 +259,6 @@ export default class ConfigGenerateCmd extends Command {
 		if (!options.fleet && options.deviceType) {
 			throw new ExpectedError(this.deviceTypeNotAllowedMessage);
 		}
-		const { normalizeOsVersion } = await import('../../utils/normalization');
-		options.version = normalizeOsVersion(options.version);
 		const { validateDevOptionAndWarn } = await import('../../utils/config');
 		await validateDevOptionAndWarn(options.dev, options.version);
 	}

lib/commands/device/os-update.ts

@@ -100,8 +100,6 @@ export default class DeviceOsUpdateCmd extends Command {
 		// Get target OS version
 		let targetOsVersion = options.version;
 		if (targetOsVersion != null) {
-			const { normalizeOsVersion } = await import('../../utils/normalization');
-			targetOsVersion = normalizeOsVersion(targetOsVersion);
 			if (!hupVersionInfo.versions.includes(targetOsVersion)) {
 				throw new ExpectedError(
 					`The provided version ${targetOsVersion} is not in the Host OS update targets for this device`,

lib/commands/os/configure.ts

@@ -216,15 +216,9 @@ export default class OsConfigureCmd extends Command {
 			configJson = JSON.parse(rawConfig);
 		}
 
-		const { normalizeOsVersion } = await import('../../utils/normalization');
-		const osVersion = normalizeOsVersion(
+		const osVersion =
 			options.version ||
-				(await getOsVersionFromImage(
-					params.image,
-					deviceTypeManifest,
-					devInit,
-				)),
-		);
+			(await getOsVersionFromImage(params.image, deviceTypeManifest, devInit));
 
 		const { validateDevOptionAndWarn } = await import('../../utils/config');
 		await validateDevOptionAndWarn(options.dev, osVersion);

lib/errors.ts

@@ -211,6 +211,7 @@ const EXPECTED_ERROR_REGEXES = [
 	/^BalenaOrganizationNotFound/, // balena-sdk
 	/Request error: Unauthorized$/, // balena-sdk
 	/^Missing \d+ required arg/, // oclif parser: RequiredArgsError
+	/Missing required flag/, // oclif parser: RequiredFlagError
 	/^Unexpected argument/, // oclif parser: UnexpectedArgsError
 	/to be one of/, // oclif parser: FlagInvalidOptionError, ArgInvalidOptionError
 	/must also be provided when using/, // oclif parser (depends-on)

lib/utils/cloud.ts

@@ -202,8 +202,12 @@ async function resolveOSVersion(
 	if (['menu', 'menu-esr'].includes(version)) {
 		return await selectOSVersionFromMenu(deviceType, version === 'menu-esr');
 	}
-	const { normalizeOsVersion } = await import('./normalization');
-	version = normalizeOsVersion(version);
+	// Note that `version` may also be 'latest', 'recommended', 'default'
+	if (/^v?\d+\.\d+\.\d+/.test(version)) {
+		if (version[0] === 'v') {
+			version = version.slice(1);
+		}
+	}
 	return version;
 }
 

lib/utils/compose.ts

@@ -20,7 +20,6 @@ import type * as SDK from 'balena-sdk';
 import type Dockerode = require('dockerode');
 import * as path from 'path';
 import type { Composition, ImageDescriptor } from '@balena/compose/dist/parse';
-import type { RetryParametersObj } from 'pinejs-client-core';
 import type {
 	BuiltImage,
 	ComposeOpts,
@@ -95,62 +94,22 @@ export function createProject(
 	};
 }
 
-const getRequestRetryParameters = (): RetryParametersObj => {
-	if (
-		process.env.BALENA_CLI_TEST_TYPE != null &&
-		process.env.BALENA_CLI_TEST_TYPE !== ''
-	) {
-		// We only read the test env vars when in test mode.
-		const { intVar } =
-			require('@balena/env-parsing') as typeof import('@balena/env-parsing');
-		// We use the BALENARCTEST namespace and only parse the env vars while in test mode
-		// since we plan to switch all pinejs clients with the one of the SDK and might not
-		// want to have to support these env vars.
-		return {
-			minDelayMs: intVar('BALENARCTEST_API_RETRY_MIN_DELAY_MS'),
-			maxDelayMs: intVar('BALENARCTEST_API_RETRY_MAX_DELAY_MS'),
-			maxAttempts: intVar('BALENARCTEST_API_RETRY_MAX_ATTEMPTS'),
-		};
-	}
-
-	return {
-		minDelayMs: 1000,
-		maxDelayMs: 60000,
-		maxAttempts: 7,
-	};
-};
-
 export const createRelease = async function (
-	logger: Logger,
 	apiEndpoint: string,
 	auth: string,
 	userId: number,
 	appId: number,
 	composition: Composition,
 	draft: boolean,
-	semver: string | undefined,
-	contract: string | undefined,
+	semver?: string,
+	contract?: string,
 ): Promise<Release> {
 	const _ = require('lodash') as typeof import('lodash');
 	const crypto = require('crypto') as typeof import('crypto');
 	const releaseMod =
 		require('@balena/compose/dist/release') as typeof import('@balena/compose/dist/release');
 
-	const client = releaseMod.createClient({
-		apiEndpoint,
-		auth,
-		retry: {
-			...getRequestRetryParameters(),
-			onRetry: (err, delayMs, attempt, maxAttempts) => {
-				const code = err?.statusCode ?? 0;
-				logger.logDebug(
-					`API call failed with code ${code}.  Attempting retry ${attempt} of ${maxAttempts} in ${
-						delayMs / 1000
-					} seconds`,
-				);
-			},
-		},
-	});
+	const client = releaseMod.createClient({ apiEndpoint, auth });
 
 	const { release, serviceImages } = await releaseMod.create({
 		client,
@@ -280,13 +239,11 @@ export const authorizePush = function (
 	tokenAuthEndpoint: string,
 	registry: string,
 	images: string[],
-	previousRepos: string[],
 ): Promise<string> {
 	if (!Array.isArray(images)) {
 		images = [images];
 	}
 
-	images.push(...previousRepos);
 	return sdk.request
 		.send({
 			baseUrl: tokenAuthEndpoint,

lib/utils/compose_ts.ts

@@ -1215,31 +1215,58 @@ export async function validateProjectDirectory(
 	return result;
 }
 
+/**
+ * While testing, pushing a release with a token of up to 125 repos (new + past) worked
+ * and resulted a token of 15967 characters. Generating a token with more repos, thus
+ * bigger token fails since the request would exceed the max allowed headers size of 16KB.
+ * We use a value slightly smaller than the max to account for unknown factors that
+ * might increase the request header size.
+ */
+const MAX_SAFE_IMAGE_REPOS_PER_TOKEN = 120;
+
 async function getTokenForPreviousRepos(
 	logger: Logger,
 	appId: number,
 	apiEndpoint: string,
 	taggedImages: TaggedImage[],
-): Promise<string> {
+): Promise<Array<[taggedImage: TaggedImage, token: string]>> {
 	logger.logDebug('Authorizing push...');
 	const { authorizePush, getPreviousRepos } = await import('./compose');
 	const sdk = getBalenaSdk();
 	const previousRepos = await getPreviousRepos(sdk, logger, appId);
 
-	const token = await authorizePush(
-		sdk,
-		apiEndpoint,
-		taggedImages[0].registry,
-		_.map(taggedImages, 'repo'),
-		previousRepos,
+	const newImageChunks = _.chunk(
+		taggedImages,
+		Math.max(MAX_SAFE_IMAGE_REPOS_PER_TOKEN - previousRepos.length, 1),
 	);
-	return token;
+
+	const imagesAndTokens: Array<[taggedImage: TaggedImage, token: string]> = [];
+	for (const newImageChunk of newImageChunks) {
+		const token = await authorizePush(
+			sdk,
+			apiEndpoint,
+			newImageChunk[0].registry,
+			// We request access to the previous repos as well, so that while pushing we have access
+			// to cross mount old-matching layers, so that we can avoid re-uploading them every time.
+			[
+				...newImageChunk.map((taggedImage) => taggedImage.repo),
+				...previousRepos,
+			],
+		);
+		imagesAndTokens.push(
+			...newImageChunk.map((taggedImage): (typeof imagesAndTokens)[number] => [
+				taggedImage,
+				token,
+			]),
+		);
+	}
+
+	return imagesAndTokens;
 }
 
 async function pushAndUpdateServiceImages(
 	docker: Dockerode,
-	token: string,
-	images: TaggedImage[],
+	imagesAndTokens: Array<[taggedImage: TaggedImage, token: string]>,
 	afterEach: (
 		serviceImage: import('@balena/compose/dist/release/models').ImageModel,
 		props: object,
@@ -1249,16 +1276,19 @@ async function pushAndUpdateServiceImages(
 	const { retry } = await import('./helpers');
 	const { pushProgressRenderer } = await import('./compose');
 	const tty = (await import('./tty'))(process.stdout);
-	const opts = { authconfig: { registrytoken: token } };
 	const progress = new DockerProgress({ docker });
 	const renderer = pushProgressRenderer(
 		tty,
 		getChalk().blue('[Push]') + '    ',
 	);
-	const reporters = progress.aggregateProgress(images.length, renderer);
+	const reporters = progress.aggregateProgress(
+		imagesAndTokens.length,
+		renderer,
+	);
 
 	const pushImage = async (
 		localImage: Dockerode.Image,
+		token: string,
 		index: number,
 	): Promise<string> => {
 		try {
@@ -1267,7 +1297,10 @@ async function pushAndUpdateServiceImages(
 			// "name": "registry2.balena-cloud.com/v2/aa27790dff571ec7d2b4fbcf3d4648d5:latest"
 			const imgName: string = (localImage as any).name || '';
 			const imageDigest: string = await retry({
-				func: () => progress.push(imgName, reporters[index], opts),
+				func: () =>
+					progress.push(imgName, reporters[index], {
+						authconfig: { registrytoken: token },
+					}),
 				maxAttempts: 3, // try calling func 3 times (max)
 				label: imgName, // label for retry log messages
 				initialDelayMs: 2000, // wait 2 seconds before the 1st retry
@@ -1285,13 +1318,16 @@ async function pushAndUpdateServiceImages(
 	};
 
 	const inspectAndPushImage = async (
-		{ serviceImage, localImage, props, logs }: TaggedImage,
+		[{ serviceImage, localImage, props, logs }, token]: [
+			TaggedImage,
+			token: string,
+		],
 		index: number,
 	) => {
 		try {
 			const [imgInfo, imgDigest] = await Promise.all([
 				localImage.inspect(),
-				pushImage(localImage, index),
+				pushImage(localImage, token, index),
 			]);
 			serviceImage.image_size = imgInfo.Size;
 			serviceImage.content_hash = imgDigest;
@@ -1317,7 +1353,7 @@ async function pushAndUpdateServiceImages(
 
 	tty.hideCursor();
 	try {
-		await Promise.all(images.map(inspectAndPushImage));
+		await Promise.all(imagesAndTokens.map(inspectAndPushImage));
 	} finally {
 		tty.showCursor();
 	}
@@ -1329,16 +1365,14 @@ async function pushServiceImages(
 	pineClient: ReturnType<
 		typeof import('@balena/compose/dist/release').createClient
 	>,
-	taggedImages: TaggedImage[],
-	token: string,
+	imagesAndTokens: Array<[taggedImage: TaggedImage, token: string]>,
 	skipLogUpload: boolean,
 ): Promise<void> {
 	const releaseMod = await import('@balena/compose/dist/release');
 	logger.logInfo('Pushing images to registry...');
 	await pushAndUpdateServiceImages(
 		docker,
-		token,
-		taggedImages,
+		imagesAndTokens,
 		async function (serviceImage) {
 			logger.logDebug(
 				`Saving image ${serviceImage.is_stored_at__image_location}`,
@@ -1385,7 +1419,6 @@ export async function deployProject(
 		`${prefix}Creating release...`,
 		() =>
 			createRelease(
-				logger,
 				apiEndpoint,
 				auth,
 				userId,
@@ -1406,7 +1439,7 @@ export async function deployProject(
 			// awaitInterruptibleTask throws SIGINTError on CTRL-C,
 			// causing the release status to be set to 'failed'
 			await awaitInterruptibleTask(async () => {
-				const token = await getTokenForPreviousRepos(
+				const imagesAndTokens = await getTokenForPreviousRepos(
 					logger,
 					appId,
 					apiEndpoint,
@@ -1416,8 +1449,7 @@ export async function deployProject(
 					docker,
 					logger,
 					pineClient,
-					taggedImages,
-					token,
+					imagesAndTokens,
 					skipLogUpload,
 				);
 			});

lib/utils/config.ts

@@ -184,9 +184,9 @@ export async function validateDevOptionAndWarn(
  * option.
  */
 export async function validateSecureBootOptionAndWarn(
-	secureBoot: boolean,
-	slug: string,
-	version: string,
+	secureBoot?: boolean,
+	slug?: string,
+	version?: string,
 	logger?: import('./logger'),
 ) {
 	if (!secureBoot) {
@@ -202,7 +202,7 @@ export async function validateSecureBootOptionAndWarn(
 	const sdk = getBalenaSdk();
 	const [osRelease] = await sdk.models.os.getAllOsVersions(slug, {
 		$select: 'contract',
-		$filter: { raw_version: version },
+		$filter: { raw_version: `${version.replace(/^v/, '')}` },
 	});
 	if (!osRelease) {
 		throw new ExpectedError(`Error: No ${version} release for ${slug}`);

lib/utils/normalization.ts

@@ -81,13 +81,3 @@ export async function disambiguateReleaseParam(
 export async function lowercaseIfSlug(s: string) {
 	return s.includes('/') ? s.toLowerCase() : s;
 }
-
-export function normalizeOsVersion(version: string) {
-	// Note that `version` may also be 'latest', 'recommended', 'default'
-	if (/^v?\d+\.\d+\.\d+/.test(version)) {
-		if (version[0] === 'v') {
-			version = version.slice(1);
-		}
-	}
-	return version;
-}

package.json

@@ -1,6 +1,6 @@
 {
   "name": "balena-cli",
-  "version": "17.4.10",
+  "version": "17.4.5",
   "description": "The official balena Command Line Interface",
   "main": "./build/app.js",
   "homepage": "https://github.com/balena-io/balena-cli",
@@ -194,11 +194,10 @@
     "typescript": "^5.3.2"
   },
   "dependencies": {
-    "@balena/compose": "^3.2.0",
+    "@balena/compose": "^3.0.5",
     "@balena/dockerignore": "^1.0.2",
-    "@balena/env-parsing": "^1.1.8",
     "@balena/es-version": "^1.0.1",
-    "@oclif/core": "^3.14.1",
+    "@oclif/core": "~3.11.0",
     "@resin.io/valid-email": "^0.1.0",
     "@sentry/node": "^6.16.1",
     "@types/fast-levenshtein": "0.0.1",
@@ -224,7 +223,7 @@
     "denymount": "^2.3.0",
     "docker-modem": "3.0.0",
     "docker-progress": "^5.1.3",
-    "dockerode": "3.3.5",
+    "dockerode": "3.3.3",
     "ejs": "^3.1.6",
     "etcher-sdk": "^8.7.0",
     "event-stream": "3.3.4",
@@ -284,6 +283,6 @@
     "windosu": "^0.3.0"
   },
   "versionist": {
-    "publishedAt": "2024-01-02T12:41:39.852Z"
+    "publishedAt": "2023-12-04T14:08:26.483Z"
   }
 }

patches/all/@oclif+core+3.11.0.patch

@@ -12,10 +12,10 @@ index 607d8dc..07ba1f2 100644
      return lines.join('\n');
  }
 diff --git a/node_modules/@oclif/core/lib/help/command.js b/node_modules/@oclif/core/lib/help/command.js
-index 0753040..c1b0f67 100644
+index c528e93..2c20760 100644
 --- a/node_modules/@oclif/core/lib/help/command.js
 +++ b/node_modules/@oclif/core/lib/help/command.js
-@@ -58,7 +58,7 @@ class CommandHelp extends formatter_1.HelpFormatter {
+@@ -45,7 +45,7 @@ class CommandHelp extends formatter_1.HelpFormatter {
          if (args.filter((a) => a.description).length === 0)
              return;
          return args.map((a) => {
@@ -23,9 +23,9 @@ index 0753040..c1b0f67 100644
 +            const name = a.required ? `<${a.name}>` : `[${a.name}]`;
              let description = a.description || '';
              if (a.default)
-                 description = `${(0, theme_1.colorize)(this.config?.theme?.flagDefaultValue, `[default: ${a.default}]`)} ${description}`;
-@@ -153,14 +153,12 @@ class CommandHelp extends formatter_1.HelpFormatter {
-             label = labels.join((0, theme_1.colorize)(this.config?.theme?.flagSeparator, flag.char ? ', ' : '  '));
+                 description = `[default: ${a.default}] ${description}`;
+@@ -137,14 +137,12 @@ class CommandHelp extends formatter_1.HelpFormatter {
+             label = labels.join(flag.char ? ', ' : '  ');
          }
          if (flag.type === 'option') {
 -            let value = flag.helpValue || (this.opts.showFlagNameInTitle ? flag.name : '<value>');
@@ -42,10 +42,10 @@ index 0753040..c1b0f67 100644
          }
          return label;
 diff --git a/node_modules/@oclif/core/lib/help/index.js b/node_modules/@oclif/core/lib/help/index.js
-index 242538a..efde8ac 100644
+index 38494f5..213b8b0 100644
 --- a/node_modules/@oclif/core/lib/help/index.js
 +++ b/node_modules/@oclif/core/lib/help/index.js
-@@ -168,11 +168,12 @@ class Help extends HelpBase {
+@@ -158,11 +158,12 @@ class Help extends HelpBase {
          }
          this.log(this.formatCommand(command));
          this.log('');
@@ -56,12 +56,12 @@ index 242538a..efde8ac 100644
              this.log('');
          }
 -        if (subCommands.length > 0) {
-+        if (subTopics.length > 0 && !SUPPRESS_SUBTOPICS) {
++        if (subCommands.length > 0 && !SUPPRESS_SUBTOPICS) {
              const aliases = [];
              const uniqueSubCommands = subCommands.filter((p) => {
                  aliases.push(...p.aliases);
 diff --git a/node_modules/@oclif/core/lib/parser/errors.js b/node_modules/@oclif/core/lib/parser/errors.js
-index 656ec6b..2bbf36b 100644
+index 51be624..768d589 100644
 --- a/node_modules/@oclif/core/lib/parser/errors.js
 +++ b/node_modules/@oclif/core/lib/parser/errors.js
 @@ -14,7 +14,8 @@ Object.defineProperty(exports, "CLIError", { enumerable: true, get: function ()
@@ -71,13 +71,13 @@ index 656ec6b..2bbf36b 100644
 -        options.message += '\nSee more help with --help';
 +        const help = options.command ? `\`${options.command} --help\`` : '--help';
 +        options.message += `\nSee more help with ${help}`;
-         super(options.message, { exit: options.exit });
+         super(options.message);
          this.parse = options.parse;
      }
 @@ -37,7 +38,8 @@ exports.InvalidArgsSpecError = InvalidArgsSpecError;
  class RequiredArgsError extends CLIParseError {
      args;
-     constructor({ args, exit, flagsWithMultiple, parse, }) {
+     constructor({ args, flagsWithMultiple, parse, }) {
 -        let message = `Missing ${args.length} required arg${args.length === 1 ? '' : 's'}`;
 +        const command = 'balena ' + parse.input.context.id.replace(/:/g, ' ');
 +        let message = `Missing ${args.length} required argument${args.length === 1 ? '' : 's'}`;
@@ -88,8 +88,20 @@ index 656ec6b..2bbf36b 100644
              message += `\n\nNote: ${flags} allow${flagsWithMultiple.length === 1 ? 's' : ''} multiple values. Because of this you need to provide all arguments before providing ${flagsWithMultiple.length === 1 ? 'that flag' : 'those flags'}.`;
              message += '\nAlternatively, you can use "--" to signify the end of the flags and the beginning of arguments.';
          }
--        super({ exit: cache_1.default.getInstance().get('exitCodes')?.requiredArgs ?? exit, message, parse });
-+        super({ exit: cache_1.default.getInstance().get('exitCodes')?.requiredArgs ?? exit, message, parse, command });
+-        super({ message, parse });
++        super({ message, parse, command });
          this.args = args;
      }
  }
+@@ -56,9 +58,10 @@ exports.RequiredArgsError = RequiredArgsError;
+ class RequiredFlagError extends CLIParseError {
+     flag;
+     constructor({ flag, parse }) {
++        const command = 'balena ' + parse.input.context.id.replace(/:/g, ' ');
+         const usage = (0, list_1.renderList)((0, help_1.flagUsages)([flag], { displayRequired: false }));
+         const message = `Missing required flag:\n${usage}`;
+-        super({ message, parse });
++        super({ message, parse, command });
+         this.flag = flag;
+     }
+ }

tests/commands/deploy.spec.ts

@@ -15,7 +15,6 @@
  * limitations under the License.
  */
 
-import { intVar } from '@balena/env-parsing';
 import type { Request as ReleaseRequest } from '@balena/compose/dist/release';
 import { expect } from 'chai';
 import { promises as fs } from 'fs';
@@ -285,25 +284,16 @@ describe('balena deploy', function () {
 		api.expectPostRelease({});
 		docker.expectGetManifestBusybox();
 
-		let failedImagePatchRequests = 0;
 		// Mock this patch HTTP request to return status code 500, in which case
 		// the release status should be saved as "failed" rather than "success"
-		const maxRequestRetries = intVar('BALENARCTEST_API_RETRY_MAX_ATTEMPTS');
-		expect(
-			maxRequestRetries,
-			'BALENARCTEST_API_RETRY_MAX_ATTEMPTS must be >= 2 for this test',
-		).to.be.greaterThanOrEqual(2);
 		api.expectPatchImage({
 			replyBody: errMsg,
 			statusCode: 500,
-			// b/c failed requests are retried
-			times: maxRequestRetries,
 			inspectRequest: (_uri, requestBody) => {
 				const imageBody = requestBody as Partial<
 					import('@balena/compose/dist/release/models').ImageModel
 				>;
 				expect(imageBody.status).to.equal('success');
-				failedImagePatchRequests++;
 			},
 		});
 		// Check that the CLI patches the release with status="failed"
@@ -334,7 +324,6 @@ describe('balena deploy', function () {
 				responseCode: 200,
 				services: ['main'],
 			});
-			expect(failedImagePatchRequests).to.equal(maxRequestRetries);
 		} finally {
 			await switchSentry(sentryStatus);
 			// @ts-expect-error claims restore does not exist
@@ -342,82 +331,6 @@ describe('balena deploy', function () {
 		}
 	});
 
-	it('should create the expected --build tar stream after retrying failing OData requests (single container)', async () => {
-		const projectPath = path.join(projectsPath, 'no-docker-compose', 'basic');
-		const expectedFiles: ExpectedTarStreamFiles = {
-			'src/.dockerignore': { fileSize: 16, type: 'file' },
-			'src/start.sh': { fileSize: 89, type: 'file' },
-			'src/windows-crlf.sh': {
-				fileSize: isWindows ? 68 : 70,
-				testStream: isWindows ? expectStreamNoCRLF : undefined,
-				type: 'file',
-			},
-			Dockerfile: { fileSize: 88, type: 'file' },
-			'Dockerfile-alt': { fileSize: 30, type: 'file' },
-		};
-		const responseFilename = 'build-POST.json';
-		const responseBody = await fs.readFile(
-			path.join(dockerResponsePath, responseFilename),
-			'utf8',
-		);
-		const expectedResponseLines = [
-			...commonResponseLines[responseFilename],
-			`[Info] No "docker-compose.yml" file found at "${projectPath}"`,
-			`[Info] Creating default composition with source: "${projectPath}"`,
-			...getDockerignoreWarn1(
-				[path.join(projectPath, 'src', '.dockerignore')],
-				'deploy',
-			),
-		];
-		if (isWindows) {
-			const fname = path.join(projectPath, 'src', 'windows-crlf.sh');
-			expectedResponseLines.push(
-				`[Info] Converting line endings CRLF -> LF for file: ${fname}`,
-			);
-		}
-
-		api.expectPostRelease({});
-		docker.expectGetManifestBusybox();
-
-		const maxRequestRetries = intVar('BALENARCTEST_API_RETRY_MAX_ATTEMPTS');
-		expect(
-			maxRequestRetries,
-			'BALENARCTEST_API_RETRY_MAX_ATTEMPTS must be >= 2 for this test',
-		).to.be.greaterThanOrEqual(2);
-		let failedImagePatchRequests = 0;
-		let succesfullImagePatchRequests = 0;
-		api
-			.optPatch(/^\/v6\/image($|[(?])/, { times: maxRequestRetries })
-			.reply((_uri, requestBody) => {
-				const imageBody = requestBody as Partial<
-					import('@balena/compose/dist/release/models').ImageModel
-				>;
-				expect(imageBody.status).to.equal('success');
-				if (failedImagePatchRequests < maxRequestRetries - 1) {
-					failedImagePatchRequests++;
-					return [500, 'Patch Image Error'];
-				}
-				succesfullImagePatchRequests++;
-				return [200, 'OK'];
-			});
-		api.expectPatchRelease({});
-		api.expectPostImageLabel();
-
-		await testDockerBuildStream({
-			commandLine: `deploy testApp --build --source ${projectPath}`,
-			dockerMock: docker,
-			expectedFilesByService: { main: expectedFiles },
-			expectedQueryParamsByService: { main: commonQueryParams },
-			expectedResponseLines,
-			projectPath,
-			responseBody,
-			responseCode: 200,
-			services: ['main'],
-		});
-		expect(failedImagePatchRequests).to.equal(maxRequestRetries - 1);
-		expect(succesfullImagePatchRequests).to.equal(1);
-	});
-
 	it('should create the expected tar stream (docker-compose, --multi-dockerignore)', async () => {
 		const projectPath = path.join(projectsPath, 'docker-compose', 'basic');
 		const service1Dockerfile = (

tests/config-tests.ts

@@ -26,11 +26,6 @@ process.env.BALENARC_NO_SENTRY = '1';
 // Like the global `--unsupported` flag
 process.env.BALENARC_UNSUPPORTED = '1';
 
-// Reduce the api request retry limits to keep the tests fast.
-process.env.BALENARCTEST_API_RETRY_MIN_DELAY_MS = '100';
-process.env.BALENARCTEST_API_RETRY_MAX_DELAY_MS = '1000';
-process.env.BALENARCTEST_API_RETRY_MAX_ATTEMPTS = '2';
-
 import * as tmp from 'tmp';
 tmp.setGracefulCleanup();
 // Use a temporary dir for tests data

tests/errors.spec.ts

@@ -131,6 +131,7 @@ describe('handleError() function', () => {
 	const messagesToMatch = [
 		'Missing 1 required argument', // oclif
 		'Missing 2 required arguments', // oclif
+		'Missing required flag', // oclif
 		'Unexpected argument', // oclif
 		'Unexpected arguments', // oclif
 		'to be one of', // oclif

tests/nock/balena-api-mock.ts

@@ -35,13 +35,11 @@ export class BalenaAPIMock extends NockMock {
 		notFound = false,
 		optional = false,
 		persist = false,
-		times = undefined as number | undefined,
 		expandArchitecture = false,
 	} = {}) {
 		const interceptor = this.optGet(/^\/v6\/application($|[(?])/, {
 			optional,
 			persist,
-			times,
 		});
 		if (notFound) {
 			interceptor.reply(200, { d: [] });
@@ -107,12 +105,10 @@ export class BalenaAPIMock extends NockMock {
 		notFound = false,
 		optional = false,
 		persist = false,
-		times = undefined as number | undefined,
 	} = {}) {
 		const interceptor = this.optGet(/^\/v6\/release($|[(?])/, {
 			persist,
 			optional,
-			times,
 		});
 		if (notFound) {
 			interceptor.reply(200, { d: [] });
@@ -137,9 +133,8 @@ export class BalenaAPIMock extends NockMock {
 		inspectRequest = this.inspectNoOp,
 		optional = false,
 		persist = false,
-		times = undefined as number | undefined,
 	}) {
-		this.optPatch(/^\/v6\/release($|[(?])/, { optional, persist, times }).reply(
+		this.optPatch(/^\/v6\/release($|[(?])/, { optional, persist }).reply(
 			statusCode,
 			this.getInspectedReplyBodyFunction(inspectRequest, replyBody),
 		);
@@ -153,9 +148,8 @@ export class BalenaAPIMock extends NockMock {
 		inspectRequest = this.inspectNoOp,
 		optional = false,
 		persist = false,
-		times = undefined as number | undefined,
 	}) {
-		this.optPost(/^\/v6\/release($|[(?])/, { optional, persist, times }).reply(
+		this.optPost(/^\/v6\/release($|[(?])/, { optional, persist }).reply(
 			statusCode,
 			this.getInspectedReplyFileFunction(
 				inspectRequest,
@@ -173,9 +167,8 @@ export class BalenaAPIMock extends NockMock {
 		inspectRequest = this.inspectNoOp,
 		optional = false,
 		persist = false,
-		times = undefined as number | undefined,
 	}) {
-		this.optPatch(/^\/v6\/image($|[(?])/, { optional, persist, times }).reply(
+		this.optPatch(/^\/v6\/image($|[(?])/, { optional, persist }).reply(
 			statusCode,
 			this.getInspectedReplyBodyFunction(inspectRequest, replyBody),
 		);

tests/nock/nock-mock.ts

@@ -21,7 +21,6 @@ import * as fs from 'fs';
 export interface ScopeOpts {
 	optional?: boolean;
 	persist?: boolean;
-	times?: number;
 }
 
 /**
@@ -53,50 +52,36 @@ export class NockMock {
 		this.expect = this.scope;
 	}
 
-	public optMethod(
-		method: 'get' | 'delete' | 'patch' | 'post',
-		uri: string | RegExp | ((uri: string) => boolean),
-		{ optional = false, persist = false, times = undefined }: ScopeOpts,
-	) {
-		let scope = this.scope;
-		if (persist) {
-			scope = scope.persist();
-		}
-		let reqInterceptor = scope[method](uri);
-		if (times != null) {
-			reqInterceptor = reqInterceptor.times(times);
-		} else if (optional) {
-			reqInterceptor = reqInterceptor.optionally();
-		}
-		return reqInterceptor;
-	}
-
 	public optGet(
 		uri: string | RegExp | ((uri: string) => boolean),
-		opts: ScopeOpts,
+		{ optional = false, persist = false }: ScopeOpts,
 	): nock.Interceptor {
-		return this.optMethod('get', uri, opts);
+		const get = (persist ? this.scope.persist() : this.scope).get(uri);
+		return optional ? get.optionally() : get;
 	}
 
 	public optDelete(
 		uri: string | RegExp | ((uri: string) => boolean),
-		opts: ScopeOpts,
+		{ optional = false, persist = false }: ScopeOpts,
 	) {
-		return this.optMethod('delete', uri, opts);
+		const del = (persist ? this.scope.persist() : this.scope).delete(uri);
+		return optional ? del.optionally() : del;
 	}
 
 	public optPatch(
 		uri: string | RegExp | ((uri: string) => boolean),
-		opts: ScopeOpts,
+		{ optional = false, persist = false }: ScopeOpts,
 	) {
-		return this.optMethod('patch', uri, opts);
+		const patch = (persist ? this.scope.persist() : this.scope).patch(uri);
+		return optional ? patch.optionally() : patch;
 	}
 
 	public optPost(
 		uri: string | RegExp | ((uri: string) => boolean),
-		opts: ScopeOpts,
+		{ optional = false, persist = false }: ScopeOpts,
 	) {
-		return this.optMethod('post', uri, opts);
+		const post = (persist ? this.scope.persist() : this.scope).post(uri);
+		return optional ? post.optionally() : post;
 	}
 
 	protected inspectNoOp(_uri: string, _requestBody: nock.Body): void {