build, deploy: Add support for multi-architecture base images

Change-type: minor
This commit is contained in:
Paulo Castro 2020-09-03 16:05:42 +01:00
parent 4fe660b3a5
commit 92f48aa2f3
9 changed files with 254 additions and 67 deletions

View File

@ -243,7 +243,7 @@ export async function buildProject(opts: {
projectPath: string;
projectName: string;
composition: Composition;
arch: string;
arch: string; // --arch option or application's architecture
deviceType: string;
emulated: boolean;
buildOpts: import('./docker').BuildOpts;
@ -265,6 +265,7 @@ export async function buildProject(opts: {
);
const renderer = await startRenderer({ imageDescriptors, ...opts });
try {
await checkDockerPlatformCompatibility(opts);
await checkBuildSecretsRequirements(opts.docker, opts.projectPath);
const needsQemu = await installQemuIfNeeded({ ...opts, imageDescriptors });
@ -907,6 +908,114 @@ export function printGitignoreWarn(
}
}
/**
* Conditionally print hint messages regarding the --emulated and --pull
* options depending on a comparison between the app architecture and the
* architecture of the CPU where Docker or balenaEngine is running.
* @param arch App architecture, or --arch flag
* @param buildOpts Build options
* @param docker Dockerode instance
* @param emulated The --emulated flag
*/
async function checkDockerPlatformCompatibility({
arch, // --arch option or application's architecture
buildOpts,
docker,
emulated,
}: {
arch: string;
buildOpts: import('./docker').BuildOpts;
docker: Dockerode;
emulated: boolean;
}) {
const semver = await import('semver');
const {
asBalenaArch,
getDockerVersion,
isCompatibleArchitecture,
} = await import('./docker');
const { platformNeedsQemu } = await import('./qemu');
const { Arch: engineArch, Version: engineVersion } = await getDockerVersion(
docker,
);
// Docker Engine versions 20.10.0 to 20.10.3 are "affected by a feature"
// whereby certain image builds are aborted with an error similar to:
// "image with reference balenalib/raspberrypi3-alpine was found
// but does not match the specified platform:
// wanted linux/arm/v7, actual: linux/amd64"
// The feature intended to enforce that a requested platform (through the
// `platform` property of the Docker Engine API `ImageBuild` request,
// as constructed by the `resin-multibuild` module) matched the image's
// manifest. However, Docker then realised that too many images had missing,
// incomplete or incorrect manifests -- including single-arch balenalib base
// images -- and did a U-turn in Docker engine version 20.10.4 and later.
// References:
// * https://github.com/docker/for-linux/issues/1170
// * https://github.com/balena-io-library/resin-rpi-raspbian/issues/104
// * https://github.com/balena-io-modules/resin-multibuild/blob/v4.10.0/lib/resolve.ts#L52
// * https://www.flowdock.com/app/rulemotion/i-cli/threads/RuSu1KiWOn62xaGy7O2sn8m8BUc
//
const svOpt = { loose: true }; // treat v19.03.15 the same as v19.3.15
if (
semver.valid(engineVersion, svOpt) &&
semver.satisfies(engineVersion, '>= 20.10.0 <= 20.10.3', svOpt)
) {
Logger.getLogger().logWarn(stripIndent`
${hr}
Docker Engine version ${engineVersion} detected. This version is affected by
an issue that causes some image builds to fail with an error similar to:
"image was found but does not match the specified platform"
If you experience that error, please take any one of the following actions:
* Upgrade Docker Engine to version 20.10.4 or later. If you are using
Docker Desktop for Mac or Windows, upgrade it to version 3.2.1 or later.
* Downgrade Docker Engine to version 19.X.X. If you are using Docker Desktop
for Mac or Windows, downgrade it to version 2.X.X.
* Downgrade the balena CLI to v12.40.3 or earlier. This would however cause
support for multi-architecture base images to be lost.
* Manually run the 'docker pull' command for all base images listed in your
Dockerfile(s) prior to executing the 'balena build' or 'balena deploy'
commands, and then do not use the balena CLI's '--pull' flag.
${hr}
`);
}
// --emulated specifically means ARM emulation on x86 CPUs, so only useful
// if the Docker daemon is running on an x86 CPU and the app is ARM
const needsEmulatedOption =
['amd64', '386'].includes(engineArch) &&
(await platformNeedsQemu(docker, emulated));
const isCompatibleArch = isCompatibleArchitecture(arch, engineArch);
const pull = !!buildOpts.pull;
// Print hints regarding the --emulated and --pull options if their usage
// is likely to be helpful based on best-effort detection.
if (
!isCompatibleArch &&
(pull !== true || (needsEmulatedOption && emulated !== true))
) {
const balenaArch = asBalenaArch(engineArch);
const msg = [
`Note: Host architecture '${balenaArch}' (where Docker or balenaEngine is running)`,
`does not match the balena application architecture '${arch}'.`,
];
// TODO: improve on `--pull` suggestion by querying the architecture of
// any cached base image and comparing it with the app architecture.
if (pull !== true) {
msg.push(
'If multiarch base images are being used, the `--pull` option may be used to',
'ensure that cached base images are pulled again for a different architecture.',
);
}
if (needsEmulatedOption && emulated !== true) {
msg.push(
'The `--emulated` option may be used to enable ARM architecture emulation',
'with QEMU during the image build.',
);
}
Logger.getLogger().logInfo(msg.join('\n '));
}
}
/**
* Check whether the "build secrets" feature is being used and, if so,
* verify that the target docker daemon is balenaEngine. If the

View File

@ -310,7 +310,7 @@ function connectToDocker(host: string, port: number): Docker {
});
}
export async function performBuilds(
async function performBuilds(
composition: Composition,
tarStream: Readable,
docker: Docker,

View File

@ -164,12 +164,85 @@ export function generateBuildOpts(options: {
return opts;
}
/** Detect whether the docker daemon is balenaEngine */
export async function isBalenaEngine(docker: dockerode): Promise<boolean> {
// dockerVersion.Engine should equal 'balena-engine' for the current/latest
// version of balenaEngine, but it was at one point (mis)spelt 'balaena':
// https://github.com/balena-os/balena-engine/pull/32/files
const dockerVersion = (await docker.version()) as BalenaEngineVersion;
const dockerVersion = await getDockerVersion(docker);
return !!(
dockerVersion.Engine && dockerVersion.Engine.match(/balena|balaena/)
// dockerVersion.Engine should be 'balena-engine' for the current
// version of balenaEngine, but at one point it was spelt 'balaena':
// https://github.com/balena-os/balena-engine/pull/32/files
(dockerVersion.Engine && dockerVersion.Engine.match(/balena|balaena/))
);
}
/** Detect whether the docker daemon is Docker Desktop (Windows or Mac) */
export async function isDockerDesktop(
docker: dockerode,
): Promise<[boolean, any]> {
// Docker Desktop (Windows and Mac) with Docker Engine 19.03 reports:
// OperatingSystem: Docker Desktop
// OSType: linux
// Docker for Mac with Docker Engine 18.06 reports:
// OperatingSystem: Docker for Mac
// OSType: linux
// On Ubuntu (standard Docker installation):
// OperatingSystem: Ubuntu 18.04.2 LTS (containerized)
// OSType: linux
// https://stackoverflow.com/questions/38223965/how-can-i-detect-if-docker-for-mac-is-installed
//
const dockerInfo = await getDockerInfo(docker);
const isDD = /(?:Docker Desktop)|(?:Docker for Mac)/i.test(
dockerInfo.OperatingSystem,
);
return [isDD, dockerInfo];
}
/**
* Convert a Docker arch identifier to a balena arch identifier.
* @param engineArch One of the GOARCH values (used by Docker) listed at:
* https://golang.org/doc/install/source#environment
*/
export function asBalenaArch(engineArch: string): string {
const archs: { [arch: string]: string } = {
arm: 'armv7hf', // could also be 'rpi' though
arm64: 'aarch64',
amd64: 'amd64',
'386': 'i386',
};
return archs[engineArch] || '';
}
/**
* Determine whether the given balena arch identifier and the given
* Docker arch identifier represent compatible architectures.
* @param balenaArch One of: rpi, armv7hf, amd64, i386
* @param engineArch One of the GOARCH values: arm, arm64, amd64, 386
*/
export function isCompatibleArchitecture(
balenaArch: string,
engineArch: string,
): boolean {
return (
(balenaArch === 'rpi' && engineArch === 'arm') ||
balenaArch === asBalenaArch(engineArch)
);
}
let cachedDockerInfo: any;
let cachedDockerVersion: BalenaEngineVersion;
export async function getDockerInfo(docker: dockerode): Promise<any> {
if (cachedDockerInfo == null) {
cachedDockerInfo = await docker.info();
}
return cachedDockerInfo;
}
export async function getDockerVersion(
docker: dockerode,
): Promise<BalenaEngineVersion> {
if (cachedDockerVersion == null) {
cachedDockerVersion = await docker.version();
}
return cachedDockerVersion;
}

View File

@ -163,7 +163,7 @@ export async function installQemuIfNeeded(
): Promise<boolean> {
// call platformNeedsQemu() regardless of whether emulation is required,
// because it logs useful information
const needsQemu = await platformNeedsQemu(docker, logger);
const needsQemu = await platformNeedsQemu(docker, emulated, logger);
if (!emulated || !needsQemu) {
return false;
}
@ -196,30 +196,27 @@ export async function installQemuIfNeeded(
* - https://stackoverflow.com/questions/55388725/run-linux-arm-container-via-qemu-binfmt-misc-on-docker-lcow
*
* @param docker Dockerode instance
* @param emulated The --emulated command-line option
* @param logger Logger instance
*/
async function platformNeedsQemu(
export async function platformNeedsQemu(
docker: Dockerode,
logger: Logger,
emulated: boolean,
logger?: Logger,
): Promise<boolean> {
const dockerInfo = await docker.info();
// Docker Desktop (Windows and Mac) with Docker Engine 19.03 reports:
// OperatingSystem: Docker Desktop
// OSType: linux
// Docker for Mac with Docker Engine 18.06 reports:
// OperatingSystem: Docker for Mac
// OSType: linux
// On Ubuntu (standard Docker installation):
// OperatingSystem: Ubuntu 18.04.2 LTS (containerized)
// OSType: linux
// https://stackoverflow.com/questions/38223965/how-can-i-detect-if-docker-for-mac-is-installed
const isDockerDesktop = /(?:Docker Desktop)|(?:Docker for Mac)/i.test(
dockerInfo.OperatingSystem,
);
if (isDockerDesktop) {
logger.logInfo(stripIndent`
Docker Desktop detected (daemon architecture: "${dockerInfo.Architecture}")
Docker itself will determine and enable architecture emulation if required,
without balena-cli intervention and regardless of the --emulated option.`);
const { isDockerDesktop } = await import('./docker');
const [isDD, dockerInfo] = await isDockerDesktop(docker);
if (logger && isDD) {
const msg = [
`Docker Desktop detected (daemon architecture: "${dockerInfo.Architecture}")`,
];
if (emulated) {
msg.push(
'The --emulated option will be ignored because Docker Desktop has built-in',
'"binfmt_misc" QEMU emulation.',
);
}
logger.logInfo(msg.join('\n '));
}
return !isDockerDesktop;
return !isDD;
}

37
npm-shrinkwrap.json generated
View File

@ -3022,9 +3022,9 @@
}
},
"tslib": {
"version": "1.13.0",
"resolved": "https://registry.npmjs.org/tslib/-/tslib-1.13.0.tgz",
"integrity": "sha512-i/6DQjL8Xf3be4K/E6Wgpekn5Qasl1usyw++dAA35Ue5orEn65VIxOA+YvNNl9HV3qv70T7CNwjODHZrLwvd1Q=="
"version": "1.14.1",
"resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz",
"integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg=="
},
"typed-error": {
"version": "2.0.0",
@ -5271,9 +5271,9 @@
"integrity": "sha512-e64Qj9+4aZzjzzFpZC7p5kmm/ccCrbLhAJplhsDXQFs87XTsXwOpH4s1Io2s90Tau/8r2j9f4l/thhDevRjzxw=="
},
"errno": {
"version": "0.1.7",
"resolved": "https://registry.npmjs.org/errno/-/errno-0.1.7.tgz",
"integrity": "sha512-MfrRBDWzIWifgq6tJj60gkAwtLNb6sQPlcFrSOflcP1aFmmruKQ2wRnze/8V6kgyz7H3FF8Npzv78mZ7XLLflg==",
"version": "0.1.8",
"resolved": "https://registry.npmjs.org/errno/-/errno-0.1.8.tgz",
"integrity": "sha512-dJ6oBr5SQ1VSd9qkk7ByRgb/1SH4JZjCHSW/mr63/QcXO9zLVxvJ6Oy13nio03rxpSnVDDjFor75SjVeZWPW/A==",
"requires": {
"prr": "~1.0.1"
}
@ -6728,9 +6728,9 @@
"integrity": "sha1-mMI9qxF1ZXuMBXPozszZGw/xjIQ="
},
"fp-ts": {
"version": "2.8.1",
"resolved": "https://registry.npmjs.org/fp-ts/-/fp-ts-2.8.1.tgz",
"integrity": "sha512-HuA/6roEliHoBgEOLCKmGRcM90e2trW/ITZZ9d9P/ra7PreqQagC3Jg6OzqWkai13KUbG90b8QO9rHPBGK/ckw=="
"version": "2.9.5",
"resolved": "https://registry.npmjs.org/fp-ts/-/fp-ts-2.9.5.tgz",
"integrity": "sha512-MiHrA5teO6t8zKArE3DdMPT/Db6v2GUt5yfWnhBTrrsVfeCJUUnV6sgFvjGNBKDmEMqVwRFkEePL7wPwqrLKKA=="
},
"fragment-cache": {
"version": "0.2.1",
@ -8477,9 +8477,9 @@
"dev": true
},
"io-ts": {
"version": "2.2.9",
"resolved": "https://registry.npmjs.org/io-ts/-/io-ts-2.2.9.tgz",
"integrity": "sha512-Q9ob1VnpwyNoMam/BO6hm2dF4uu+to8NWSZNsRW6Q2Ni38PadgLZSQDo0hW7CJFgpJkQw4BXGwXzjr7c47c+fw=="
"version": "2.2.15",
"resolved": "https://registry.npmjs.org/io-ts/-/io-ts-2.2.15.tgz",
"integrity": "sha512-ww2ZPrErx5pjCCI/tWRwjlEIDEndnN9kBIxAylXj+WNIH4ZVgaUqFuabGouehkRuvrmvzO5OnZmLf+o50h4izQ=="
},
"io-ts-reporters": {
"version": "1.2.2",
@ -13468,9 +13468,9 @@
}
},
"resin-docker-build": {
"version": "1.1.5",
"resolved": "https://registry.npmjs.org/resin-docker-build/-/resin-docker-build-1.1.5.tgz",
"integrity": "sha512-Ri9bzY9mGO6Ctw5MO6EUsQNl1jMSQ6dKg4z6acE7hvxiWjNxUUqbA0Qwu8rfVU+vSswFUy8LCjcQOD9XkrNcDA==",
"version": "1.1.6",
"resolved": "https://registry.npmjs.org/resin-docker-build/-/resin-docker-build-1.1.6.tgz",
"integrity": "sha512-657lmKN1SEbaALSb5n1Mr11fze/msSOKH2aFOPBb+L7BxueC7nat5FZ0Jv07ZD0GDTiJo5Z885l6tegMC5+eaQ==",
"requires": {
"@types/bluebird": "^3.5.30",
"@types/dockerode": "^2.5.24",
@ -13581,9 +13581,9 @@
}
},
"resin-multibuild": {
"version": "4.7.2",
"resolved": "https://registry.npmjs.org/resin-multibuild/-/resin-multibuild-4.7.2.tgz",
"integrity": "sha512-2Nn3wN09uQRuDrR0uOkK7bCKheSZ94rpY6ePt7IBVyxw/6EE0GfqSj/3y2l4lxzMFRfT5K4VDHlj5DUiNCKYkA==",
"version": "4.10.0",
"resolved": "https://registry.npmjs.org/resin-multibuild/-/resin-multibuild-4.10.0.tgz",
"integrity": "sha512-Eti1HGSzTAUmpQErD9Oz0NAj9yGFzWSM3aRDMGlK1W3GVggZsI+CZ/lM3+0ffMmPbDtkQxWqBoEtP3jUl36rIw==",
"requires": {
"@types/bluebird": "^3.5.32",
"@types/dockerode": "^2.5.34",
@ -13603,6 +13603,7 @@
"resin-bundle-resolve": "^4.3.0",
"resin-compose-parse": "^2.1.2",
"resin-docker-build": "^1.1.5",
"semver": "^7.3.2",
"tar-stream": "^2.1.3",
"tar-utils": "^2.1.0",
"typed-error": "^3.2.1"

View File

@ -256,7 +256,7 @@
"resin-compose-parse": "^2.1.2",
"resin-doodles": "^0.1.1",
"resin-image-fs": "^5.0.9",
"resin-multibuild": "^4.7.2",
"resin-multibuild": "^4.10.0",
"resin-stream-logger": "^0.1.2",
"rimraf": "^3.0.2",
"semver": "^7.3.2",

View File

@ -38,8 +38,6 @@ const commonResponseLines: { [key: string]: string[] } = {
'build-POST.json': [
'[Info] Building for amd64/nuc',
'[Info] Docker Desktop detected (daemon architecture: "x86_64")',
'[Info] Docker itself will determine and enable architecture emulation if required,',
'[Info] without balena-cli intervention and regardless of the --emulated option.',
'[Success] Build succeeded!',
],
};
@ -48,6 +46,7 @@ const commonQueryParams = {
t: '${tag}',
buildargs: {},
labels: '',
platform: 'linux/amd64',
};
const commonComposeQueryParams = {
@ -57,6 +56,7 @@ const commonComposeQueryParams = {
MY_VAR_2: 'Also a variable',
},
labels: '',
platform: 'linux/amd64',
};
const hr =
@ -76,7 +76,10 @@ describe('balena build', function () {
api.expectGetWhoAmI({ optional: true, persist: true });
api.expectGetMixpanel({ optional: true });
docker.expectGetPing();
docker.expectGetVersion();
// Docker version is cached by the CLI, hence optional: true
// Docker version is also called by resin-multibuild, hence persist: true
docker.expectGetVersion({ optional: true, persist: true });
docker.expectGetImages();
});
this.afterEach(() => {
@ -122,7 +125,7 @@ describe('balena build', function () {
);
}
}
docker.expectGetInfo({});
docker.expectGetInfo({ optional: true }); // cached, hence optional
await testDockerBuildStream({
commandLine: `build ${projectPath} --deviceType nuc --arch amd64 -g`,
dockerMock: docker,
@ -178,7 +181,7 @@ describe('balena build', function () {
);
}
}
docker.expectGetInfo({});
docker.expectGetInfo({ optional: true }); // cached, hence optional
await testDockerBuildStream({
commandLine: `build ${projectPath} --deviceType nuc --arch amd64 -B BARG1=b1 -B barg2=B2 --cache-from my/img1,my/img2`,
dockerMock: docker,
@ -273,6 +276,8 @@ describe('balena build', function () {
...qemuMod,
copyQemu: async () => '',
});
// Forget cached values by re-requiring the modules
mock.reRequire('../../build/utils/docker');
mock.reRequire('../../build/utils/qemu');
docker.expectGetInfo({ OperatingSystem: 'balenaOS 2.44.0+rev1' });
await testDockerBuildStream({
@ -280,7 +285,10 @@ describe('balena build', function () {
dockerMock: docker,
expectedFilesByService: { main: expectedFiles },
expectedQueryParamsByService: {
main: Object.entries(commonQueryParams),
main: Object.entries({
...commonQueryParams,
platform: 'linux/arm/v6',
}),
},
expectedResponseLines,
projectPath,
@ -291,6 +299,8 @@ describe('balena build', function () {
} finally {
mock.stop(fsModPath);
mock.stop(qemuModPath);
// Forget cached values by re-requiring the modules
mock.reRequire('../../build/utils/docker');
}
});
@ -334,7 +344,7 @@ describe('balena build', function () {
'[Warn] Windows-format line endings were detected in some files, but were not converted due to `--noconvert-eol` option.',
);
}
docker.expectGetInfo({});
docker.expectGetInfo({ optional: true }); // cached, hence optional
await testDockerBuildStream({
commandLine: `build ${projectPath} --deviceType nuc --arch amd64 --noconvert-eol -m`,
dockerMock: docker,
@ -430,7 +440,7 @@ describe('balena build', function () {
)}`,
);
}
docker.expectGetInfo({});
docker.expectGetInfo({ optional: true }); // cached, hence optional
await testDockerBuildStream({
commandLine: `build ${projectPath} --deviceType nuc --arch amd64 --convert-eol -G -B COMPOSE_ARG=A -B barg=b --cache-from my/img1,my/img2`,
dockerMock: docker,
@ -516,7 +526,7 @@ describe('balena build', function () {
)}`,
);
}
docker.expectGetInfo({});
docker.expectGetInfo({ optional: true }); // cached, hence optional
await testDockerBuildStream({
commandLine: `build ${projectPath} --deviceType nuc --arch amd64 --convert-eol -m`,
dockerMock: docker,

View File

@ -37,9 +37,6 @@ const commonResponseLines = {
'build-POST.json': [
'[Info] Building for armv7hf/raspberrypi3',
'[Info] Docker Desktop detected (daemon architecture: "x86_64")',
'[Info] Docker itself will determine and enable architecture emulation if required,',
'[Info] without balena-cli intervention and regardless of the --emulated option.',
// '[Build] main Step 1/4 : FROM busybox',
'[Info] Creating release...',
'[Info] Pushing images to registry...',
'[Info] Saving release...',
@ -52,6 +49,7 @@ const commonQueryParams = [
['t', '${tag}'],
['buildargs', '{}'],
['labels', ''],
['platform', 'linux/arm/v7'],
];
const commonComposeQueryParams = {
@ -61,6 +59,7 @@ const commonComposeQueryParams = {
MY_VAR_2: 'Also a variable',
},
labels: '',
platform: 'linux/arm/v7',
};
const hr =
@ -89,8 +88,10 @@ describe('balena deploy', function () {
docker.expectGetImages();
docker.expectGetPing();
docker.expectGetInfo({});
docker.expectGetVersion({ persist: true });
// optional because docker.info() and docker.version() are cached
docker.expectGetInfo({ optional: true });
// docker.version() is also called by resin-multibuild, hence persist: true
docker.expectGetVersion({ optional: true, persist: true });
docker.expectPostImagesTag();
docker.expectPostImagesPush();
docker.expectDeleteImages();
@ -307,7 +308,6 @@ describe('balena deploy', function () {
);
}
// docker.expectGetImages();
api.expectPatchImage({});
api.expectPatchRelease({});

View File

@ -195,9 +195,6 @@ export async function testDockerBuildStream(o: {
inspectTarStream(buildRequestBody, expectedFiles, projectPath),
tag,
});
if (o.commandLine.startsWith('build')) {
o.dockerMock.expectGetImages();
}
}
resetDockerignoreCache();