ExternalVendorCode/balena-cli
1
0
Fork 0
mirror of https://github.com/balena-io/balena-cli.git synced 2024-12-18 21:27:51 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge pull request #2834 from balena-io/remove-image-manager

Browse Source 
Embed balena-image-manager instead of having it as a dependency
This commit is contained in:
Matthew Yarmolinsky 2024-09-20 13:47:48 -04:00 committed by GitHub
commit a402dffbc5
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
9 changed files with 976 additions and 100 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

184
npm-shrinkwrap.json generated
View File

@ -20,7 +20,6 @@
"balena-device-init": "^7.0.1",
"balena-errors": "^4.7.3",
"balena-image-fs": "^7.0.6",
"balena-image-manager": "^10.0.1",
"balena-preload": "^15.0.6",
"balena-sdk": "^19.7.3",
"balena-semver": "^2.3.0",
@ -56,6 +55,8 @@
"JSONStream": "^1.0.3",
"livepush": "^3.5.1",
"lodash": "^4.17.21",
"mime": "^2.4.6",
"mkdirp": "^3.0.1",
"ndjson": "^2.0.0",
"node-cleanup": "^2.1.2",
"node-unzip-2": "^0.2.8",
@ -116,9 +117,12 @@
"@types/jsonwebtoken": "^9.0.6",
"@types/klaw": "^3.0.6",
"@types/lodash": "^4.14.178",
"@types/mime": "^3.0.4",
"@types/mixpanel": "^2.14.3",
"@types/mocha": "^10.0.7",
"@types/mock-fs": "^4.13.4",
"@types/mock-require": "^2.0.1",
"@types/mockery": "^1.4.33",
"@types/ndjson": "^2.0.1",
"@types/node": "^20.0.0",
"@types/node-cleanup": "^2.1.2",
@ -156,15 +160,17 @@
"intercept-stdout": "^0.1.2",
"jsonwebtoken": "^9.0.0",
"klaw": "^4.1.0",
"mkdirp": "^3.0.1",
"mocha": "^10.6.0",
"mock-fs": "^5.2.0",
"mock-require": "^3.0.3",
"mockery": "^2.1.0",
"nock": "^13.2.1",
"oclif": "^4.14.0",
"parse-link-header": "^2.0.0",
"rewire": "^7.0.0",
"simple-git": "^3.14.1",
"sinon": "^18.0.0",
"string-to-stream": "^3.0.1",
"ts-node": "^10.4.0",
"typescript": "^5.6.2"
},
@ -3820,9 +3826,9 @@
"integrity": "sha512-OpXEVoCKSS3lQqjx9GGGOapBeuW5eUboYHRlHP9urXPX25IKZ6AnP5ZRxtVf63iieUbsHxLn8NQ5Nlftc6yzAA=="
},
"node_modules/@types/mime": {
"version": "1.3.2",
"resolved": "https://registry.npmjs.org/@types/mime/-/mime-1.3.2.tgz",
"integrity": "sha512-YATxVxgRqNH6nHEIsvg6k2Boc1JHI9ZbH5iWFFv/MTkchz3b1ieGDa5T0a9RznNdI0KhVbdbWSN+KWWrQZRxTw==",
"version": "3.0.4",
"resolved": "https://registry.npmjs.org/@types/mime/-/mime-3.0.4.tgz",
"integrity": "sha512-iJt33IQnVRkqeqC7PzBHPTC6fDlRNRW8vjrgqtScAhrmMwe8c4Eo7+fUGTa+XdWrpEgpyKWMYmi2dIwMAYRzPw==",
"dev": true
},
"node_modules/@types/minimatch": {
@ -3843,6 +3849,15 @@
"integrity": "sha512-GN8yJ1mNTcFcah/wKEFIJckJx9iJLoMSzWcfRRuxz/Jk+U6KQNnml+etbtxFK8lPjzOw3zp4Ha/kjSst9fsHYw==",
"dev": true
},
"node_modules/@types/mock-fs": {
"version": "4.13.4",
"resolved": "https://registry.npmjs.org/@types/mock-fs/-/mock-fs-4.13.4.tgz",
"integrity": "sha512-mXmM0o6lULPI8z3XNnQCpL0BGxPwx1Ul1wXYEPBGl4efShyxW2Rln0JOPEWGyZaYZMM6OVXM/15zUuFMY52ljg==",
"dev": true,
"dependencies": {
"@types/node": "*"
}
},
"node_modules/@types/mock-require": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/@types/mock-require/-/mock-require-2.0.1.tgz",
@ -3852,6 +3867,12 @@
"@types/node": "*"
}
},
"node_modules/@types/mockery": {
"version": "1.4.33",
"resolved": "https://registry.npmjs.org/@types/mockery/-/mockery-1.4.33.tgz",
"integrity": "sha512-vpuuVxCnCEM0OakYNoyFs40mjJFJFJahBHyx0Z0Piysof+YwlDJzNO4V1weRvYySAmtAvlb0UHtxVO2IfTcykw==",
"dev": true
},
"node_modules/@types/mute-stream": {
"version": "0.0.4",
"resolved": "https://registry.npmjs.org/@types/mute-stream/-/mute-stream-0.0.4.tgz",
@ -4004,6 +4025,12 @@
"@types/node": "*"
}
},
"node_modules/@types/serve-static/node_modules/@types/mime": {
"version": "1.3.5",
"resolved": "https://registry.npmjs.org/@types/mime/-/mime-1.3.5.tgz",
"integrity": "sha512-/pyBZWSLD2n0dcHE3hq8s8ZvcETHtEuF+3E7XVt0Ig2nvsVQXdghHVcEkIWjy9A0wKfTn97a/PSDYohKIlnP/w==",
"dev": true
},
"node_modules/@types/shell-escape": {
"version": "0.2.0",
"resolved": "https://registry.npmjs.org/@types/shell-escape/-/shell-escape-0.2.0.tgz",
@ -5447,6 +5474,15 @@
"node": ">=18"
}
},
"node_modules/balena-device-init/node_modules/string-to-stream": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/string-to-stream/-/string-to-stream-1.1.1.tgz",
"integrity": "sha512-QySF2+3Rwq0SdO3s7BAp4x+c3qsClpPQ6abAmb0DGViiSBAkT5kL6JT2iyzEVP+T1SmzHrQD1TwlP9QAHCc+Sw==",
"dependencies": {
"inherits": "^2.0.1",
"readable-stream": "^2.1.0"
}
},
"node_modules/balena-errors": {
"version": "4.9.0",
"resolved": "https://registry.npmjs.org/balena-errors/-/balena-errors-4.9.0.tgz",
@ -5487,87 +5523,6 @@
"node": ">=16"
}
},
"node_modules/balena-image-manager": {
"version": "10.0.1",
"resolved": "https://registry.npmjs.org/balena-image-manager/-/balena-image-manager-10.0.1.tgz",
"integrity": "sha512-HUMH6NZdKfJFgNYk9K7GVN6OLbt/aY4POOx+atu2tG6cRZnuaClDYGxwPS9+oWEoBdjD3MHkcjfDa3rW1niUsQ==",
"dependencies": {
"balena-sdk": "^19.0.1",
"mime": "^2.4.6",
"mkdirp": "^1.0.4",
"rimraf": "^5.0.5"
},
"engines": {
"node": ">=18.0.0"
}
},
"node_modules/balena-image-manager/node_modules/brace-expansion": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz",
"integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==",
"dependencies": {
"balanced-match": "^1.0.0"
}
},
"node_modules/balena-image-manager/node_modules/glob": {
"version": "10.4.5",
"resolved": "https://registry.npmjs.org/glob/-/glob-10.4.5.tgz",
"integrity": "sha512-7Bv8RF0k6xjo7d4A/PxYLbUCfb6c+Vpd2/mB2yRDlew7Jb5hEXiCD9ibfO7wpk8i4sevK6DFny9h7EYbM3/sHg==",
"dependencies": {
"foreground-child": "^3.1.0",
"jackspeak": "^3.1.2",
"minimatch": "^9.0.4",
"minipass": "^7.1.2",
"package-json-from-dist": "^1.0.0",
"path-scurry": "^1.11.1"
},
"bin": {
"glob": "dist/esm/bin.mjs"
},
"funding": {
"url": "https://github.com/sponsors/isaacs"
}
},
"node_modules/balena-image-manager/node_modules/minimatch": {
"version": "9.0.5",
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.5.tgz",
"integrity": "sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow==",
"license": "ISC",
"dependencies": {
"brace-expansion": "^2.0.1"
},
"engines": {
"node": ">=16 || 14 >=14.17"
},
"funding": {
"url": "https://github.com/sponsors/isaacs"
}
},
"node_modules/balena-image-manager/node_modules/mkdirp": {
"version": "1.0.4",
"resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-1.0.4.tgz",
"integrity": "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==",
"bin": {
"mkdirp": "bin/cmd.js"
},
"engines": {
"node": ">=10"
}
},
"node_modules/balena-image-manager/node_modules/rimraf": {
"version": "5.0.10",
"resolved": "https://registry.npmjs.org/rimraf/-/rimraf-5.0.10.tgz",
"integrity": "sha512-l0OE8wL34P4nJH/H2ffoaniAokM2qSmrtXHmlpvYr5AVVX8msAyW0l8NVJFDxlSK4u3Uh/f41cQheDVdnYijwQ==",
"dependencies": {
"glob": "^10.3.7"
},
"bin": {
"rimraf": "dist/esm/bin.mjs"
},
"funding": {
"url": "https://github.com/sponsors/isaacs"
}
},
"node_modules/balena-preload": {
"version": "15.0.6",
"resolved": "https://registry.npmjs.org/balena-preload/-/balena-preload-15.0.6.tgz",
@ -5670,6 +5625,7 @@
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/mime/-/mime-3.0.0.tgz",
"integrity": "sha512-jSCU7/VB1loIWBZe14aEYHU/+1UMEHoaO7qxCOVJOw9GgH72VAWppxNcjU+x9a2k3GSIBXNKxXQFqRvvZ7vr3A==",
"license": "MIT",
"bin": {
"mime": "cli.js"
},
@ -12458,6 +12414,7 @@
"version": "2.6.0",
"resolved": "https://registry.npmjs.org/mime/-/mime-2.6.0.tgz",
"integrity": "sha512-USPkMeET31rOMiarsBNIHZKLGgvKc/LrjofAnBlOttf5ajRvqiRA8QsenbcooctK6d6Ts6aqZXBA+XbkKthiQg==",
"license": "MIT",
"bin": {
"mime": "cli.js"
},
@ -12534,7 +12491,6 @@
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-3.0.1.tgz",
"integrity": "sha512-+NsyUUAZDmo6YVHzL/stxSu3t9YS1iljliy3BSDrXJ/dkn1KYdmtZODGGjLcc9XLgVVpH4KshHB8XmZgMhaBXg==",
"dev": true,
"bin": {
"mkdirp": "dist/cjs/src/bin.js"
},
@ -12683,6 +12639,15 @@
"node": ">=10"
}
},
"node_modules/mock-fs": {
"version": "5.2.0",
"resolved": "https://registry.npmjs.org/mock-fs/-/mock-fs-5.2.0.tgz",
"integrity": "sha512-2dF2R6YMSZbpip1V1WHKGLNjr/k48uQClqMVb5H3MOvwc9qhYis3/IWbj02qIg/Y8MDXKFF4c5v0rxx2o6xTZw==",
"dev": true,
"engines": {
"node": ">=12.0.0"
}
},
"node_modules/mock-require": {
"version": "3.0.3",
"resolved": "https://registry.npmjs.org/mock-require/-/mock-require-3.0.3.tgz",
@ -12714,6 +12679,12 @@
"node": ">=0.10.0"
}
},
"node_modules/mockery": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/mockery/-/mockery-2.1.0.tgz",
"integrity": "sha512-9VkOmxKlWXoDO/h1jDZaS4lH33aWfRiJiNT/tKj+8OGzrcFDLo8d0syGdbsc3Bc4GvRXPb+NMMvojotmuGJTvA==",
"dev": true
},
"node_modules/moment": {
"version": "2.30.1",
"resolved": "https://registry.npmjs.org/moment/-/moment-2.30.1.tgz",
@ -14577,9 +14548,9 @@
}
},
"node_modules/prebuild-install/node_modules/node-abi": {
"version": "3.67.0",
"resolved": "https://registry.npmjs.org/node-abi/-/node-abi-3.67.0.tgz",
"integrity": "sha512-bLn/fU/ALVBE9wj+p4Y21ZJWYFjUXLXPi/IewyLZkx3ApxKDNBWCKdReeKOtD8dWpOdDCeMyLh6ZewzcLsG2Nw==",
"version": "3.68.0",
"resolved": "https://registry.npmjs.org/node-abi/-/node-abi-3.68.0.tgz",
"integrity": "sha512-7vbj10trelExNjFSBm5kTvZXXa7pZyKWx9RCKIyqe6I9Ev3IzGpQoqBP3a+cOdxY+pWj6VkP28n/2wWysBHD/A==",
"dependencies": {
"semver": "^7.3.5"
},
@ -15557,6 +15528,15 @@
"string-to-stream": "^1.0.1"
}
},
"node_modules/rindle/node_modules/string-to-stream": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/string-to-stream/-/string-to-stream-1.1.1.tgz",
"integrity": "sha512-QySF2+3Rwq0SdO3s7BAp4x+c3qsClpPQ6abAmb0DGViiSBAkT5kL6JT2iyzEVP+T1SmzHrQD1TwlP9QAHCc+Sw==",
"dependencies": {
"inherits": "^2.0.1",
"readable-stream": "^2.1.0"
}
},
"node_modules/roarr": {
"version": "2.15.4",
"resolved": "https://registry.npmjs.org/roarr/-/roarr-2.15.4.tgz",
@ -16428,12 +16408,26 @@
}
},
"node_modules/string-to-stream": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/string-to-stream/-/string-to-stream-1.1.1.tgz",
"integrity": "sha512-QySF2+3Rwq0SdO3s7BAp4x+c3qsClpPQ6abAmb0DGViiSBAkT5kL6JT2iyzEVP+T1SmzHrQD1TwlP9QAHCc+Sw==",
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/string-to-stream/-/string-to-stream-3.0.1.tgz",
"integrity": "sha512-Hl092MV3USJuUCC6mfl9sPzGloA3K5VwdIeJjYIkXY/8K+mUvaeEabWJgArp+xXrsWxCajeT2pc4axbVhIZJyg==",
"dev": true,
"dependencies": {
"inherits": "^2.0.1",
"readable-stream": "^2.1.0"
"readable-stream": "^3.4.0"
}
},
"node_modules/string-to-stream/node_modules/readable-stream": {
"version": "3.6.2",
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz",
"integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==",
"dev": true,
"dependencies": {
"inherits": "^2.0.3",
"string_decoder": "^1.1.1",
"util-deprecate": "^1.0.1"
},
"engines": {
"node": ">= 6"
}
},
"node_modules/string-width": {

10
package.json
View File

@ -138,9 +138,12 @@
"@types/jsonwebtoken": "^9.0.6",
"@types/klaw": "^3.0.6",
"@types/lodash": "^4.14.178",
"@types/mime": "^3.0.4",
"@types/mixpanel": "^2.14.3",
"@types/mocha": "^10.0.7",
"@types/mock-fs": "^4.13.4",
"@types/mock-require": "^2.0.1",
"@types/mockery": "^1.4.33",
"@types/ndjson": "^2.0.1",
"@types/node": "^20.0.0",
"@types/node-cleanup": "^2.1.2",
@ -178,15 +181,17 @@
"intercept-stdout": "^0.1.2",
"jsonwebtoken": "^9.0.0",
"klaw": "^4.1.0",
"mkdirp": "^3.0.1",
"mocha": "^10.6.0",
"mock-fs": "^5.2.0",
"mock-require": "^3.0.3",
"mockery": "^2.1.0",
"nock": "^13.2.1",
"oclif": "^4.14.0",
"parse-link-header": "^2.0.0",
"rewire": "^7.0.0",
"simple-git": "^3.14.1",
"sinon": "^18.0.0",
"string-to-stream": "^3.0.1",
"ts-node": "^10.4.0",
"typescript": "^5.6.2"
},
@ -201,7 +206,6 @@
"balena-device-init": "^7.0.1",
"balena-errors": "^4.7.3",
"balena-image-fs": "^7.0.6",
"balena-image-manager": "^10.0.1",
"balena-preload": "^15.0.6",
"balena-sdk": "^19.7.3",
"balena-semver": "^2.3.0",
@ -237,6 +241,8 @@
"JSONStream": "^1.0.3",
"livepush": "^3.5.1",
"lodash": "^4.17.21",
"mime": "^2.4.6",
"mkdirp": "^3.0.1",
"ndjson": "^2.0.0",
"node-cleanup": "^2.1.2",
"node-unzip-2": "^0.2.8",

2
src/commands/os/download.ts
View File

@ -89,7 +89,7 @@ export default class OsDownloadCmd extends Command {
// balenaOS ESR versions require user authentication
if (options.version) {
const { isESR } = await import('balena-image-manager');
const { isESR } = await import('../../utils/image-manager');
if (options.version === 'menu-esr' || isESR(options.version)) {
try {
await OsDownloadCmd.checkLoggedIn();

4
src/utils/cloud.ts
View File

@ -145,8 +145,8 @@ export async function downloadOSImage(
// some ongoing issues with the os download stream.
process.env.ZLIB_FLUSH = 'Z_NO_FLUSH';
const manager = await import('balena-image-manager');
const stream = await manager.get(deviceType, OSVersion);
const { getStream } = await import('./image-manager');
const stream = await getStream(deviceType, OSVersion);
const displayVersion = await new Promise((resolve, reject) => {
stream.on('error', reject);

299
src/utils/image-manager.ts Normal file
View File

@ -0,0 +1,299 @@
/**
* @license
* Copyright 2019 Balena Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import type * as SDK from 'balena-sdk';
import { getBalenaSdk } from './lazy';
// eslint-disable-next-line no-useless-escape
const BALENAOS_VERSION_REGEX = /v?\d+\.\d+\.\d+(\.rev\d+)?((\-|\+).+)?/;
/**
* @summary Check if the string is a valid balenaOS version number
* @description Throws an error if the version is invalid
*
* @param {String} version - version number to validate
* @returns {void} the most recent compatible version.
*/
const validateVersion = (version: string) => {
if (!BALENAOS_VERSION_REGEX.test(version)) {
throw new Error('Invalid version number');
}
};
/**
* @summary Get file created date
*
* @param {String} filePath - file path
* @returns {Promise<Date>} date since creation
*
* @example
* getFileCreatedDate('foo/bar').then (createdTime) ->
* console.log("The file was created in #{createdTime}")
*/
export const getFileCreatedDate = async (filePath: string) => {
const { promises: fs } = await import('fs');
const { ctime } = await fs.stat(filePath);
return ctime;
};
/**
* @summary Get path to image in cache
*
* @param {String} deviceType - device type slug or alias
* @param {String} version - the exact balenaOS version number
* @returns {Promise<String>} image path
*
* @example
* getImagePath('raspberry-pi', '1.2.3').then (imagePath) ->
* console.log(imagePath)
*/
export const getImagePath = async (deviceType: string, version?: string) => {
if (typeof version === 'string') {
validateVersion(version);
}
const balena = getBalenaSdk();
const [cacheDirectory, deviceTypeInfo] = await Promise.all([
balena.settings.get('cacheDirectory'),
balena.models.config.getDeviceTypeManifestBySlug(deviceType),
]);
const extension = deviceTypeInfo.yocto.fstype === 'zip' ? 'zip' : 'img';
const path = await import('path');
return path.join(cacheDirectory, `${deviceType}-v${version}.${extension}`);
};
/**
* @summary Determine if a device image is fresh
*
* @description
* If the device image does not exist, return false.
*
* @param {String} deviceType - device type slug or alias
* @param {String} version - the exact balenaOS version number
* @returns {Promise<Boolean>} is image fresh
*
* @example
* isImageFresh('raspberry-pi', '1.2.3').then (isFresh) ->
* if isFresh
* console.log('The Raspberry Pi image v1.2.3 is fresh!')
*/
export const isImageFresh = async (deviceType: string, version: string) => {
const imagePath = await getImagePath(deviceType, version);
let createdDate;
try {
createdDate = await getFileCreatedDate(imagePath);
} catch {
// Swallow errors from getFileCreatedTime.
}
if (createdDate == null) {
return false;
}
const balena = getBalenaSdk();
const lastModifiedDate = await balena.models.os.getLastModified(
deviceType,
version,
);
return lastModifiedDate < createdDate;
};
/**
* Heuristically determine whether the given semver version is a balenaOS
* ESR version.
*
* @param {string} version Semver version. If invalid or range, return false.
*/
export const isESR = (version: string) => {
const match = version.match(/^v?(\d+)\.\d+\.\d+/);
const major = parseInt((match && match[1]) || '', 10);
return major >= 2018; // note: (NaN >= 2018) is false
};
/**
* @summary Get the most recent compatible version
*
* @param {String} deviceType - device type slug or alias
* @param {String} versionOrRange - supports the same version options
* as `balena.models.os.getMaxSatisfyingVersion`.
* See `getStream` for the detailed explanation.
* @returns {Promise<String>} the most recent compatible version.
*/
const resolveVersion = async (deviceType: string, versionOrRange: string) => {
const balena = getBalenaSdk();
const version = await balena.models.os.getMaxSatisfyingVersion(
deviceType,
versionOrRange,
isESR(versionOrRange) ? 'esr' : 'default',
);
if (!version) {
throw new Error('No such version for the device type');
}
return version;
};
/**
* @summary Get an image from the cache
*
* @param {String} deviceType - device type slug or alias
* @param {String} version - the exact balenaOS version number
* @returns {Promise<fs.ReadStream>} image readable stream
*
* @example
* getImage('raspberry-pi', '1.2.3').then (stream) ->
* stream.pipe(fs.createWriteStream('foo/bar.img'))
*/
export const getImage = async (deviceType: string, version: string) => {
const imagePath = await getImagePath(deviceType, version);
const fs = await import('fs');
const stream = fs.createReadStream(imagePath) as ReturnType<
typeof fs.createReadStream
> & { mime: string };
// Default to application/octet-stream if we could not find a more specific mime type
const { getType } = await import('mime');
stream.mime = getType(imagePath) ?? 'application/octet-stream';
return stream;
};
/**
* @summary Get a writable stream for an image in the cache
*
* @param {String} deviceType - device type slug or alias
* @param {String} version - the exact balenaOS version number
* @returns {Promise<fs.WriteStream & { persistCache: () => Promise<void>, removeCache: () => Promise<void> }>} image writable stream
*
* @example
* getImageWritableStream('raspberry-pi', '1.2.3').then (stream) ->
* fs.createReadStream('foo/bar').pipe(stream)
*/
export const getImageWritableStream = async (
deviceType: string,
version?: string,
) => {
const imagePath = await getImagePath(deviceType, version);
// Ensure the cache directory exists, to prevent
// ENOENT errors when trying to write to it.
const path = await import('path');
const { mkdirp } = await import('mkdirp');
await mkdirp(path.dirname(imagePath));
// Append .inprogress to streams, move them to the right location only on success
const inProgressPath = imagePath + '.inprogress';
const { promises, createWriteStream } = await import('fs');
type ImageWritableStream = ReturnType<typeof createWriteStream> &
Record<'persistCache' | 'removeCache', () => Promise<void>>;
const stream = createWriteStream(inProgressPath) as ImageWritableStream;
// Call .isCompleted on the stream
stream.persistCache = () => promises.rename(inProgressPath, imagePath);
stream.removeCache = () => promises.unlink(inProgressPath);
return stream;
};
type DownloadConfig = NonNullable<
Parameters<SDK.BalenaSDK['models']['os']['download']>[0]
>;
const doDownload = async (options: DownloadConfig) => {
const balena = getBalenaSdk();
const imageStream = await balena.models.os.download(options);
// Piping to a PassThrough stream is needed to be able
// to then pipe the stream to multiple destinations.
const { PassThrough } = await import('stream');
const pass = new PassThrough();
imageStream.pipe(pass);
// Save a copy of the image in the cache
const cacheStream = await getImageWritableStream(
options.deviceType,
options.version,
);
pass.pipe(cacheStream, { end: false });
pass.on('end', cacheStream.persistCache);
// If we return `pass` directly, the client will not be able
// to read all data from it after a delay, since it will be
// instantly piped to `cacheStream`.
// The solution is to create yet another PassThrough stream,
// pipe to it and return the new stream instead.
const pass2 = new PassThrough() as InstanceType<typeof PassThrough> & {
mime: string;
};
pass2.mime = imageStream.mime;
imageStream.on('progress', (state) => pass2.emit('progress', state));
imageStream.on('error', async (err) => {
await cacheStream.removeCache();
pass2.emit('error', err);
});
return pass.pipe(pass2);
};
/**
* @summary Get a device operating system image
* @public
*
* @description
* This function saves a copy of the downloaded image in the cache directory setting specified in [balena-settings-client](https://github.com/balena-io-modules/balena-settings-client).
*
* @param {String} deviceType - device type slug or alias
* @param {String} versionOrRange - can be one of
* * the exact version number,
* in which case it is used if the version is supported,
* or the promise is rejected,
* * a [semver](https://www.npmjs.com/package/semver)-compatible
* range specification, in which case the most recent satisfying version is used
* if it exists, or the promise is rejected,
* * `'latest'` in which case the most recent version is used, including pre-releases,
* * `'recommended'` in which case the recommended version is used, i.e. the most
* recent version excluding pre-releases, the promise is rejected
* if only pre-release versions are available,
* * `'default'` in which case the recommended version is used if available,
* or `latest` is used otherwise.
* Defaults to `'latest'`.
* @param {Object} options
* @param {boolean} options?.developmentMode
* @returns {Promise<NodeJS.ReadableStream>} image readable stream
*
* @example
* getStream('raspberry-pi', 'default').then (stream) ->
* stream.pipe(fs.createWriteStream('foo/bar.img'))
*/
export const getStream = async (
deviceType: string,
versionOrRange?: string,
options: Omit<DownloadConfig, 'deviceType' | 'version'> = {},
) => {
if (versionOrRange == null) {
versionOrRange = 'latest';
}
const version = await resolveVersion(deviceType, versionOrRange);
const isFresh = await isImageFresh(deviceType, version);
const $stream = isFresh
? await getImage(deviceType, version)
: await doDownload({ ...options, deviceType, version });
// schedule the 'version' event for the next iteration of the event loop
// so that callers have a chance of adding an event handler
setImmediate(() =>
$stream.emit('balena-image-manager:resolved-version', version),
);
return $stream;
};

5
tests/utils.ts Normal file
View File

@ -0,0 +1,5 @@
export async function delay(ms: number) {
await new Promise((resolve) => {
setTimeout(resolve, ms);
});
}

1
tests/utils/image-manager/fixtures/lorem.txt Normal file
View File

@ -0,0 +1 @@
Lorem ipsum dolor sit amet

BIN
tests/utils/image-manager/fixtures/lorem.zip Normal file
View File

Binary file not shown.

571
tests/utils/image-manager/image-manager.spec.ts Normal file
View File

@ -0,0 +1,571 @@
import * as stream from 'stream';
import { AssertionError, expect } from 'chai';
import { stub } from 'sinon';
import * as tmp from 'tmp';
import { delay } from '../../utils';
import * as fs from 'fs';
import * as fsAsync from 'fs/promises';
import * as stringToStream from 'string-to-stream';
import { Writable as WritableStream } from 'stream';
import * as imageManager from '../../../build/utils/image-manager';
import { resolve, extname } from 'path';
import * as mockFs from 'mock-fs';
import * as rimraf from 'rimraf';
import { promisify } from 'util';
import * as os from 'os';
// Make sure we're all using literally the same instance of balena-sdk
// so we can mock out methods called by the real code
import { getBalenaSdk } from '../../../build/utils/lazy';
const balena = getBalenaSdk();
const fsExistsAsync = promisify(fs.exists);
const clean = async () => {
await promisify(rimraf)(await balena.settings.get('cacheDirectory'));
};
describe('image-manager', function () {
describe('.getStream()', () => {
describe('given the existing image', function () {
beforeEach(function () {
this.image = tmp.fileSync();
fs.writeSync(this.image.fd, 'Cache image', 0, 'utf8');
this.cacheGetImagePathStub = stub(imageManager, 'getImagePath');
return this.cacheGetImagePathStub.returns(
Promise.resolve(this.image.name),
);
});
afterEach(function () {
this.cacheGetImagePathStub.restore();
return this.image.removeCallback();
});
describe('given the image is fresh', function () {
beforeEach(function () {
this.cacheIsImageFresh = stub(imageManager, 'isImageFresh');
return this.cacheIsImageFresh.returns(Promise.resolve(true));
});
afterEach(function () {
return this.cacheIsImageFresh.restore();
});
it('should eventually become a readable stream of the cached image', function (done) {
this.timeout(5000);
void imageManager.getStream('raspberry-pi').then(function (stream) {
let result = '';
stream.on('data', (chunk) => (result += chunk.toString()));
return stream.on('end', function () {
expect(result).to.equal('Cache image');
return done();
});
});
});
});
describe('given the image is not fresh', function () {
beforeEach(function () {
this.cacheIsImageFresh = stub(imageManager, 'isImageFresh');
return this.cacheIsImageFresh.returns(Promise.resolve(false));
});
afterEach(function () {
return this.cacheIsImageFresh.restore();
});
describe('given a valid download endpoint', function () {
beforeEach(function () {
this.osDownloadStub = stub(balena.models.os, 'download');
this.osDownloadStub.returns(
Promise.resolve(stringToStream('Download image')),
);
});
afterEach(function () {
this.osDownloadStub.restore();
});
it('should eventually become a readable stream of the download image and save a backup copy', function (done) {
if (os.platform() === 'win32') {
// Skipping test on Windows because we get `EPERM: operation not permitted, rename` for `getImageWritableStream` on the windows runner
this.skip();
}
void imageManager.getStream('raspberry-pi').then((stream) => {
let result = '';
stream.on('data', (chunk) => (result += chunk));
stream.on('end', async () => {
expect(result).to.equal('Download image');
const contents = await fsAsync.readFile(this.image.name, {
encoding: 'utf8',
});
expect(contents).to.equal('Download image');
done();
});
});
});
it('should be able to read from the stream after a slight delay', function (done) {
void imageManager.getStream('raspberry-pi').then(async (s) => {
await delay(200);
const pass = new stream.PassThrough();
s.pipe(pass);
let result = '';
pass.on('data', (chunk) => (result += chunk));
pass.on('end', function () {
expect(result).to.equal('Download image');
done();
});
});
});
});
describe('given a failing download', function () {
beforeEach(function () {
this.osDownloadStream = new stream.PassThrough();
this.osDownloadStub = stub(balena.models.os, 'download');
this.osDownloadStub.returns(Promise.resolve(this.osDownloadStream));
});
afterEach(function () {
this.osDownloadStub.restore();
});
it('should clean up the in progress cached stream if an error occurs', function (done) {
if (os.platform() === 'win32') {
// Skipping test on Windows because we get `EPERM: operation not permitted, rename` for `getImageWritableStream` on the windows runner
this.skip();
}
void imageManager.getStream('raspberry-pi').then((stream) => {
stream.on('data', () => {
// After the first chunk, error
return this.osDownloadStream.emit('error');
});
stream.on('error', async () => {
const contents = await fsAsync
.stat(this.image.name + '.inprogress')
.then(function () {
throw new AssertionError(
'Image cache should be deleted on failure',
);
})
.catch((err) => {
if (err.code !== 'ENOENT') {
throw err;
}
return fsAsync.readFile(this.image.name, {
encoding: 'utf8',
});
});
expect(contents).to.equal('Cache image');
done();
});
stringToStream('Download image').pipe(this.osDownloadStream);
});
});
});
describe('given a stream with the mime property', async function () {
beforeEach(function () {
this.osDownloadStub = stub(balena.models.os, 'download');
const message = 'Lorem ipsum dolor sit amet';
const mockResultStream = stringToStream(message) as ReturnType<
typeof stringToStream
> & {
mime?: string;
};
mockResultStream.mime = 'application/zip';
this.osDownloadStub.returns(Promise.resolve(mockResultStream));
});
afterEach(function () {
this.osDownloadStub.restore();
});
it('should preserve the property', () =>
imageManager
.getStream('raspberry-pi')
.then((resultStream) =>
expect(resultStream.mime).to.equal('application/zip'),
));
});
});
});
});
describe('.getImagePath()', () => {
describe('given a cache directory', function () {
beforeEach(function () {
this.balenaSettingsGetStub = stub(balena.settings, 'get');
this.balenaSettingsGetStub
.withArgs('cacheDirectory')
.returns(
Promise.resolve(
os.platform() === 'win32'
? 'C:\\Users\\johndoe\\_balena\\cache'
: '/Users/johndoe/.balena/cache',
),
);
});
afterEach(function () {
this.balenaSettingsGetStub.restore();
});
describe('given valid slugs', function () {
beforeEach(function () {
this.getDeviceTypeManifestBySlugStub = stub(
balena.models.config,
'getDeviceTypeManifestBySlug',
);
this.getDeviceTypeManifestBySlugStub.withArgs('raspberry-pi').returns(
Promise.resolve({
yocto: {
fstype: 'resin-sdcard',
},
}),
);
this.getDeviceTypeManifestBySlugStub.withArgs('intel-edison').returns(
Promise.resolve({
yocto: {
fstype: 'zip',
},
}),
);
});
afterEach(function () {
this.getDeviceTypeManifestBySlugStub.restore();
});
it('should eventually equal an absolute path', async () => {
await imageManager
.getImagePath('raspberry-pi', '1.2.3')
.then(function (imagePath) {
const isAbsolute = imagePath === resolve(imagePath);
expect(isAbsolute).to.be.true;
});
});
it('should eventually equal the correct path', async function () {
const result = await imageManager.getImagePath(
'raspberry-pi',
'1.2.3',
);
expect(result).to.equal(
os.platform() === 'win32'
? 'C:\\Users\\johndoe\\_balena\\cache\\raspberry-pi-v1.2.3.img'
: '/Users/johndoe/.balena/cache/raspberry-pi-v1.2.3.img',
);
});
it('should use a zip extension for directory images', async () => {
const imagePath = await imageManager.getImagePath(
'intel-edison',
'1.2.3',
);
expect(extname(imagePath)).to.equal('.zip');
});
it('given invalid version should be rejected', async function () {
const promise = imageManager.getImagePath('intel-edison', 'DOUGH');
await expect(promise).to.be.eventually.rejectedWith(
'Invalid version number',
);
});
});
});
});
describe('.isImageFresh()', () => {
describe('given the raspberry-pi manifest', function () {
beforeEach(function () {
this.getDeviceTypeManifestBySlugStub = stub(
balena.models.config,
'getDeviceTypeManifestBySlug',
);
this.getDeviceTypeManifestBySlugStub.returns(
Promise.resolve({
yocto: {
fstype: 'balena-sdcard',
},
}),
);
});
afterEach(function () {
this.getDeviceTypeManifestBySlugStub.restore();
});
describe('given the file does not exist', function () {
beforeEach(function () {
this.utilsGetFileCreatedDate = stub(
imageManager,
'getFileCreatedDate',
);
this.utilsGetFileCreatedDate.returns(
Promise.reject(new Error("ENOENT, stat 'raspberry-pi'")),
);
});
afterEach(function () {
this.utilsGetFileCreatedDate.restore();
});
it('should return false', async function () {
expect(await imageManager.isImageFresh('raspberry-pi', '1.2.3')).to.be
.false;
});
});
describe('given a fixed created time', function () {
beforeEach(function () {
this.utilsGetFileCreatedDate = stub(
imageManager,
'getFileCreatedDate',
);
this.utilsGetFileCreatedDate.returns(
Promise.resolve(new Date('2014-01-01T00:00:00.000Z')),
);
});
afterEach(function () {
this.utilsGetFileCreatedDate.restore();
});
describe('given the file was created before the os last modified time', function () {
beforeEach(function () {
this.osGetLastModified = stub(balena.models.os, 'getLastModified');
this.osGetLastModified.returns(
Promise.resolve(new Date('2014-02-01T00:00:00.000Z')),
);
});
afterEach(function () {
this.osGetLastModified.restore();
});
it('should return false', function () {
const promise = imageManager.isImageFresh('raspberry-pi', '1.2.3');
return expect(promise).to.eventually.be.false;
});
});
describe('given the file was created after the os last modified time', function () {
beforeEach(function () {
this.osGetLastModified = stub(balena.models.os, 'getLastModified');
this.osGetLastModified.returns(
Promise.resolve(new Date('2013-01-01T00:00:00.000Z')),
);
});
afterEach(function () {
this.osGetLastModified.restore();
});
it('should return true', function () {
const promise = imageManager.isImageFresh('raspberry-pi', '1.2.3');
return expect(promise).to.eventually.be.true;
});
});
describe('given the file was created just at the os last modified time', function () {
beforeEach(function () {
this.osGetLastModified = stub(balena.models.os, 'getLastModified');
this.osGetLastModified.returns(
Promise.resolve(new Date('2014-00-01T00:00:00.000Z')),
);
});
afterEach(function () {
this.osGetLastModified.restore();
});
it('should return false', function () {
const promise = imageManager.isImageFresh('raspberry-pi', '1.2.3');
return expect(promise).to.eventually.be.false;
});
});
});
});
});
describe('.getImage()', () => {
describe('given an existing image', function () {
beforeEach(function () {
this.image = tmp.fileSync();
fs.writeSync(this.image.fd, 'Lorem ipsum dolor sit amet', 0, 'utf8');
this.cacheGetImagePathStub = stub(imageManager, 'getImagePath');
this.cacheGetImagePathStub.returns(Promise.resolve(this.image.name));
});
afterEach(function (done) {
this.cacheGetImagePathStub.restore();
fs.unlink(this.image.name, done);
});
it('should return a stream to the image', function (done) {
void imageManager
.getImage('lorem-ipsum', '1.2.3')
.then(function (stream) {
let result = '';
stream.on('data', (chunk) => (result += chunk));
stream.on('end', function () {
expect(result).to.equal('Lorem ipsum dolor sit amet');
done();
});
});
});
it('should contain the mime property', () =>
imageManager
.getImage('lorem-ipsum', '1.2.3')
.then((stream) =>
expect(stream.mime).to.equal('application/octet-stream'),
));
});
});
describe('.getImageWritableStream()', () => {
describe('given the valid image path', function () {
beforeEach(function () {
this.image = tmp.fileSync();
this.cacheGetImagePathStub = stub(imageManager, 'getImagePath');
this.cacheGetImagePathStub.returns(Promise.resolve(this.image.name));
});
afterEach(function (done) {
this.cacheGetImagePathStub.restore();
fs.unlink(this.image.name, done);
});
it('should return a writable stream', () =>
imageManager
.getImageWritableStream('raspberry-pi', '1.2.3')
.then((stream) =>
expect(stream).to.be.an.instanceof(WritableStream),
));
it('should allow writing to the stream', function (done) {
if (os.platform() === 'win32') {
// Skipping test on Windows because we get `EPERM: operation not permitted, rename` for `getImageWritableStream` on the windows runner
this.skip();
}
void imageManager
.getImageWritableStream('raspberry-pi', '1.2.3')
.then((stream) => {
const stringStream = stringToStream('Lorem ipsum dolor sit amet');
stringStream.pipe(stream);
stream.on('finish', async () => {
await stream.persistCache();
const contents = await fsAsync.readFile(this.image.name, {
encoding: 'utf8',
});
expect(contents).to.equal('Lorem ipsum dolor sit amet');
done();
});
});
});
});
});
describe('.getFileCreatedDate()', function () {
describe('given the file exists', function () {
beforeEach(function () {
this.date = new Date(2014, 1, 1);
this.fsStatStub = stub(fs.promises, 'stat');
this.fsStatStub
.withArgs('foo')
.returns(Promise.resolve({ ctime: this.date }));
});
afterEach(function () {
this.fsStatStub.restore();
});
it('should eventually equal the created time in milliseconds', async function () {
const promise = imageManager.getFileCreatedDate('foo');
await expect(promise).to.eventually.equal(this.date);
});
});
describe('given the file does not exist', function () {
beforeEach(function () {
this.fsStatStub = stub(fs.promises, 'stat');
this.fsStatStub
.withArgs('foo')
.returns(Promise.reject(new Error("ENOENT, stat 'foo'")));
});
afterEach(function () {
this.fsStatStub.restore();
});
it('should be rejected with an error', async function () {
const promise = imageManager.getFileCreatedDate('foo');
await expect(promise).to.be.rejectedWith('ENOENT');
});
});
});
describe('.clean()', function () {
describe('given the cache with saved images', function () {
beforeEach(async function () {
this.cacheDirectory = await balena.settings.get('cacheDirectory');
mockFs({
[this.cacheDirectory]: {
'raspberry-pi': 'Raspberry Pi Image',
'intel-edison': 'Intel Edison Image',
parallela: 'Parallela Image',
},
});
});
afterEach(() => {
mockFs.restore();
});
it('should remove the cache directory completely', async function () {
const exists = await fsExistsAsync(this.cacheDirectory);
expect(exists).to.be.true;
await clean();
const exists2 = await fsExistsAsync(this.cacheDirectory);
expect(exists2).to.be.false;
});
});
describe('given no cache', function () {
beforeEach(async function () {
this.cacheDirectory = await balena.settings.get('cacheDirectory');
mockFs({});
});
afterEach(() => mockFs.restore());
it('should keep the cache directory removed', async function () {
const exists = await fsExistsAsync(this.cacheDirectory);
expect(exists).to.be.false;
await clean();
const exists2 = await fsExistsAsync(this.cacheDirectory);
expect(exists2).to.be.false;
});
});
});
});