ExternalVendorCode/balena-cli
1
0
Fork 0
mirror of https://github.com/balena-io/balena-cli.git synced 2024-12-18 21:27:51 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

build, deploy: Add support for multiarch base images

Browse Source 
Bump version of balena-multibuild to the one that supports multiarch
Remove previous hack to avoid sending platform information to multibuild

Change-type: minor
Signed-off-by: Paul Jonathan <pj@balena.io>
See: https://github.com/balena-io/balena-cli/issues/1508
This commit is contained in:
toochevere 2021-07-28 13:19:57 +00:00
parent 56c1af50c0
commit ffccbfba12
13 changed files with 212 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
lib/utils/compose_ts.ts
View File

@ -1051,9 +1051,6 @@ export async function makeBuildTasks(
infoStr = `build [${task.context}]`;
}
logger.logDebug(` ${task.serviceName}: ${infoStr}`);
// Workaround for Docker v20.10 + single-arch base images. See:
// https://www.flowdock.com/app/rulemotion/i-cli/threads/RuSu1KiWOn62xaGy7O2sn8m8BUc
task.dockerPlatform = 'none';
});
logger.logDebug(

35
npm-shrinkwrap.json generated
View File

@ -6497,6 +6497,14 @@
}
}
},
"dockerfile-ast": {
"version": "0.2.1",
"resolved": "https://registry.npmjs.org/dockerfile-ast/-/dockerfile-ast-0.2.1.tgz",
"integrity": "sha512-ut04CVM1G6zIITTcYPDIXhPZk9mCa21m4dfW8FcDDGxwgTQhYyHDu6U7M8klZ7QsjqVcJhryKi+TGOX6bjgKdQ==",
"requires": {
"vscode-languageserver-types": "^3.16.0"
}
},
"dockerfile-template": {
"version": "0.2.0",
"resolved": "https://registry.npmjs.org/dockerfile-template/-/dockerfile-template-0.2.0.tgz",
@ -8202,9 +8210,9 @@
"integrity": "sha1-mMI9qxF1ZXuMBXPozszZGw/xjIQ="
},
"fp-ts": {
"version": "2.10.5",
"resolved": "https://registry.npmjs.org/fp-ts/-/fp-ts-2.10.5.tgz",
"integrity": "sha512-X2KfTIV0cxIk3d7/2Pvp/pxL/xr2MV1WooyEzKtTWYSc1+52VF4YzjBTXqeOlSiZsPCxIBpDGfT9Dyo7WEY0DQ=="
"version": "2.11.2",
"resolved": "https://registry.npmjs.org/fp-ts/-/fp-ts-2.11.2.tgz",
"integrity": "sha512-G1rD89nmbbgTNRBKohjB3Qv4IxOHQ5KV3ZvYfpaQZyrGt+ZQUFrcnCqE567bcEdvwoAUKDQM7isOcv7xcM/qAQ=="
},
"fragment-cache": {
"version": "0.2.1",
@ -15556,9 +15564,12 @@
}
},
"@types/klaw": {
"version": "1.3.5",
"resolved": "https://registry.npmjs.org/@types/klaw/-/klaw-1.3.5.tgz",
"integrity": "sha512-KZfv4ea6bEbdQhfwpxtDuTPO2mHAAXMQqPOZyS4MgNyCymKoLHp0FVzzYq3H2zCeIotN4h1453TahLCCm8rf2w=="
"version": "1.3.6",
"resolved": "https://registry.npmjs.org/@types/klaw/-/klaw-1.3.6.tgz",
"integrity": "sha512-4pr2RxwhfsLxFYa4Ip8JxrdXIvPX7fAqyBh9ofZPedMwf8M5CIcSQskqvX6/5Y/zpCBHtuC3218t8H+XJsg5FA==",
"requires": {
"@types/node": "*"
}
},
"bl": {
"version": "1.2.3",
@ -15755,13 +15766,14 @@
}
},
"resin-multibuild": {
"version": "4.11.0",
"resolved": "https://registry.npmjs.org/resin-multibuild/-/resin-multibuild-4.11.0.tgz",
"integrity": "sha512-rIYV9GDNuI8pU9N+wGdVRIOGAnw1BFdbyt3BkvERFxbf+b/e7jpBjHkbK8VPQdRMlKPyu137ZxQlR3z7EivJBg==",
"version": "4.12.1",
"resolved": "https://registry.npmjs.org/resin-multibuild/-/resin-multibuild-4.12.1.tgz",
"integrity": "sha512-ORtzaDZGS5wftNo4KXi4yOcqSsjU0/56oY7mXlc8XcmqusOOfr1N3rnFpXkjQ7COJLcPvfPT+OEeJuQ7l7cOmg==",
"requires": {
"ajv": "^6.12.3",
"bluebird": "^3.7.2",
"docker-progress": "^5.0.0",
"dockerfile-ast": "^0.2.1",
"dockerfile-template": "^0.2.0",
"dockerode": "^2.5.8",
"fp-ts": "^2.8.1",
@ -18527,6 +18539,11 @@
}
}
},
"vscode-languageserver-types": {
"version": "3.16.0",
"resolved": "https://registry.npmjs.org/vscode-languageserver-types/-/vscode-languageserver-types-3.16.0.tgz",
"integrity": "sha512-k8luDIWJWyenLc5ToFQQMaSrqCHiLwyKPHKPQZ5zz21vM+vIVUSvsRpcbiECH4WR88K2XZqc4ScRcZ7nk/jbeA=="
},
"wcwidth": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/wcwidth/-/wcwidth-1.0.1.tgz",

3
package.json
View File

@ -66,6 +66,7 @@
"test:standalone": "npm run build:standalone && npm run test:standalone:fast",
"test:standalone:fast": "cross-env BALENA_CLI_TEST_TYPE=standalone mocha --config .mocharc-standalone.js",
"test:fast": "npm run build:fast && npm run test:source",
"test:debug": "cross-env BALENA_CLI_TEST_TYPE=source mocha --inspect-brk=0.0.0.0",
"test:only": "npm run build:fast && cross-env BALENA_CLI_TEST_TYPE=source mocha \"tests/**/${npm_config_test}.spec.ts\"",
"catch-uncommitted": "ts-node --transpile-only automation/run.ts catch-uncommitted",
"ci": "npm run test && npm run catch-uncommitted",
@ -267,7 +268,7 @@
"resin-cli-visuals": "^1.8.0",
"resin-compose-parse": "^2.1.3",
"resin-doodles": "^0.1.1",
"resin-multibuild": "^4.11.0",
"resin-multibuild": "^4.12.1",
"resin-stream-logger": "^0.1.2",
"rimraf": "^3.0.2",
"semver": "^7.3.2",

54
tests/commands/build.spec.ts
View File

@ -53,6 +53,16 @@ const commonQueryParams = {
labels: '',
};
const commonQueryParamsIntel = {
...commonQueryParams,
platform: 'linux/amd64',
};
const commonQueryParamsArmV6 = {
...commonQueryParams,
platform: 'linux/arm/v6',
};
const commonComposeQueryParams = {
t: '${tag}',
buildargs: {
@ -62,6 +72,11 @@ const commonComposeQueryParams = {
labels: '',
};
const commonComposeQueryParamsIntel = {
...commonComposeQueryParams,
platform: 'linux/amd64',
};
// "itSS" means "it() Skip Standalone"
const itSS = process.env.BALENA_CLI_TEST_TYPE === 'standalone' ? it.skip : it;
@ -76,7 +91,7 @@ describe('balena build', function () {
api.expectGetWhoAmI({ optional: true, persist: true });
api.expectGetMixpanel({ optional: true });
docker.expectGetPing();
docker.expectGetVersion();
docker.expectGetVersion({ persist: true });
});
this.afterEach(() => {
@ -123,13 +138,16 @@ describe('balena build', function () {
}
}
docker.expectGetInfo({});
docker.expectGetManifestBusybox();
await testDockerBuildStream({
commandLine: `build ${projectPath} --deviceType nuc --arch amd64 ${
isV13() ? '' : '-g'
}`,
dockerMock: docker,
expectedFilesByService: { main: expectedFiles },
expectedQueryParamsByService: { main: Object.entries(commonQueryParams) },
expectedQueryParamsByService: {
main: Object.entries(commonQueryParamsIntel),
},
expectedResponseLines,
projectPath,
responseBody,
@ -152,7 +170,7 @@ describe('balena build', function () {
'Dockerfile-alt': { fileSize: 30, type: 'file' },
};
const expectedQueryParams = {
...commonQueryParams,
...commonQueryParamsIntel,
buildargs: '{"BARG1":"b1","barg2":"B2"}',
cachefrom: '["my/img1","my/img2"]',
};
@ -181,6 +199,7 @@ describe('balena build', function () {
}
}
docker.expectGetInfo({});
docker.expectGetManifestBusybox();
await testDockerBuildStream({
commandLine: `build ${projectPath} --deviceType nuc --arch amd64 -B BARG1=b1 -B barg2=B2 --cache-from my/img1,my/img2`,
dockerMock: docker,
@ -271,6 +290,7 @@ describe('balena build', function () {
});
mock.reRequire('../../build/utils/qemu');
docker.expectGetInfo({ OperatingSystem: 'balenaOS 2.44.0+rev1' });
docker.expectGetManifestBusybox();
await testDockerBuildStream({
commandLine: `build ${projectPath} --emulated --deviceType ${deviceType} --arch ${arch} ${
isV13() ? '' : '--nogitignore'
@ -278,7 +298,7 @@ describe('balena build', function () {
dockerMock: docker,
expectedFilesByService: { main: expectedFiles },
expectedQueryParamsByService: {
main: Object.entries(commonQueryParams),
main: Object.entries(commonQueryParamsArmV6),
},
expectedResponseLines,
projectPath,
@ -327,11 +347,15 @@ describe('balena build', function () {
);
}
docker.expectGetInfo({});
docker.expectGetManifestBusybox();
await testDockerBuildStream({
commandLine: `build ${projectPath} --deviceType nuc --arch amd64 --noconvert-eol -m`,
dockerMock: docker,
expectedFilesByService: { main: expectedFiles },
expectedQueryParamsByService: { main: Object.entries(commonQueryParams) },
expectedQueryParamsByService: {
main: Object.entries(commonQueryParamsIntel),
},
expectedResponseLines,
projectPath,
responseBody,
@ -360,7 +384,7 @@ describe('balena build', function () {
},
service2: {
'.dockerignore': { fileSize: 12, type: 'file' },
'Dockerfile-alt': { fileSize: 40, type: 'file' },
'Dockerfile-alt': { fileSize: 13, type: 'file' },
'file2-crlf.sh': {
fileSize: isWindows ? 12 : 14,
testStream: isWindows ? expectStreamNoCRLF : undefined,
@ -386,7 +410,7 @@ describe('balena build', function () {
}),
),
service2: Object.entries(
_.merge({}, commonComposeQueryParams, {
_.merge({}, commonComposeQueryParamsIntel, {
buildargs: {
COMPOSE_ARG: 'A',
barg: 'b',
@ -417,6 +441,8 @@ describe('balena build', function () {
);
}
docker.expectGetInfo({});
docker.expectGetManifestNucAlpine();
docker.expectGetManifestBusybox();
await testDockerBuildStream({
commandLine: `build ${projectPath} --deviceType nuc --arch amd64 --convert-eol ${
isV13() ? '' : '-G'
@ -453,7 +479,7 @@ describe('balena build', function () {
},
service2: {
'.dockerignore': { fileSize: 12, type: 'file' },
'Dockerfile-alt': { fileSize: 40, type: 'file' },
'Dockerfile-alt': { fileSize: 13, type: 'file' },
'file2-crlf.sh': {
fileSize: isWindows ? 12 : 14,
testStream: isWindows ? expectStreamNoCRLF : undefined,
@ -473,7 +499,7 @@ describe('balena build', function () {
}),
),
service2: Object.entries(
_.merge({}, commonComposeQueryParams, {
_.merge({}, commonComposeQueryParamsIntel, {
buildargs: {
COMPOSE_ARG: 'an argument defined in the docker-compose.yml file',
},
@ -505,6 +531,9 @@ describe('balena build', function () {
);
}
docker.expectGetInfo({});
docker.expectGetManifestBusybox();
docker.expectGetManifestNucAlpine();
await testDockerBuildStream({
commandLine: `build ${projectPath} --deviceType nuc --arch amd64 --convert-eol -m`,
dockerMock: docker,
@ -539,7 +568,7 @@ describe('balena build', function () {
},
service2: {
'.dockerignore': { fileSize: 12, type: 'file' },
'Dockerfile-alt': { fileSize: 40, type: 'file' },
'Dockerfile-alt': { fileSize: 13, type: 'file' },
'file2-crlf.sh': {
fileSize: isWindows ? 12 : 14,
testStream: isWindows ? expectStreamNoCRLF : undefined,
@ -559,7 +588,7 @@ describe('balena build', function () {
}),
),
service2: Object.entries(
_.merge({}, commonComposeQueryParams, {
_.merge({}, commonComposeQueryParamsIntel, {
buildargs: {
COMPOSE_ARG: 'an argument defined in the docker-compose.yml file',
},
@ -593,6 +622,9 @@ describe('balena build', function () {
const projectName = 'spectest';
const tag = 'myTag';
docker.expectGetInfo({});
docker.expectGetManifestBusybox();
docker.expectGetManifestNucAlpine();
await testDockerBuildStream({
commandLine: `build ${projectPath} --deviceType nuc --arch amd64 --convert-eol -m --tag ${tag} --projectName ${projectName}`,
dockerMock: docker,

16
tests/commands/deploy.spec.ts
View File

@ -53,6 +53,7 @@ const commonResponseLines = {
};
const commonQueryParams = [
['platform', 'linux/arm/v7'],
['t', '${tag}'],
['buildargs', '{}'],
['labels', ''],
@ -67,6 +68,11 @@ const commonComposeQueryParams = {
labels: '',
};
const commonComposeQueryParamsArmV7 = {
...commonComposeQueryParams,
platform: 'linux/arm/v7',
};
describe('balena deploy', function () {
let api: BalenaAPIMock;
let docker: DockerMock;
@ -139,6 +145,7 @@ describe('balena deploy', function () {
api.expectPatchImage({});
api.expectPatchRelease({});
api.expectPostImageLabel();
docker.expectGetManifestBusybox();
await testDockerBuildStream({
commandLine: `deploy testApp --build --source ${projectPath} ${
@ -189,6 +196,7 @@ describe('balena deploy', function () {
api.expectPatchImage({});
api.expectPatchRelease({});
api.expectPostImageLabel();
docker.expectGetManifestBusybox();
await testDockerBuildStream({
commandLine: `deploy testApp --build --source ${projectPath}`,
@ -238,6 +246,7 @@ describe('balena deploy', function () {
api.expectPatchImage({});
api.expectPatchRelease({});
api.expectPostImageLabel();
docker.expectGetManifestBusybox();
await testDockerBuildStream({
commandLine: `deploy testApp --build --draft --source ${projectPath}`,
@ -275,6 +284,7 @@ describe('balena deploy', function () {
const expectedExitCode = 1;
api.expectPostRelease({});
docker.expectGetManifestBusybox();
// Mock this patch HTTP request to return status code 500, in which case
// the release status should be saved as "failed" rather than "success"
@ -352,7 +362,7 @@ describe('balena deploy', function () {
},
service2: {
'.dockerignore': { fileSize: 12, type: 'file' },
'Dockerfile-alt': { fileSize: 40, type: 'file' },
'Dockerfile-alt': { fileSize: 13, type: 'file' },
'file2-crlf.sh': {
fileSize: isWindows ? 12 : 14,
testStream: isWindows ? expectStreamNoCRLF : undefined,
@ -372,7 +382,7 @@ describe('balena deploy', function () {
}),
),
service2: Object.entries(
_.merge({}, commonComposeQueryParams, {
_.merge({}, commonComposeQueryParamsArmV7, {
buildargs: {
COMPOSE_ARG: 'an argument defined in the docker-compose.yml file',
},
@ -407,6 +417,8 @@ describe('balena deploy', function () {
api.expectPostRelease({});
api.expectPatchImage({});
api.expectPatchRelease({});
docker.expectGetManifestRpi3Alpine();
docker.expectGetManifestBusybox();
await testDockerBuildStream({
commandLine: `deploy testApp --build --source ${projectPath} --multi-dockerignore`,

4
tests/commands/push.spec.ts
View File

@ -455,7 +455,7 @@ describe('balena push', function () {
'docker-compose.yml': { fileSize: 332, type: 'file' },
'service1/Dockerfile.template': { fileSize: 144, type: 'file' },
'service1/file1.sh': { fileSize: 12, type: 'file' },
'service2/Dockerfile-alt': { fileSize: 40, type: 'file' },
'service2/Dockerfile-alt': { fileSize: 13, type: 'file' },
'service2/.dockerignore': { fileSize: 12, type: 'file' },
'service2/file2-crlf.sh': {
fileSize: isWindows ? 12 : 14,
@ -508,7 +508,7 @@ describe('balena push', function () {
'service1/Dockerfile.template': { fileSize: 144, type: 'file' },
'service1/file1.sh': { fileSize: 12, type: 'file' },
'service1/test-ignore.txt': { fileSize: 12, type: 'file' },
'service2/Dockerfile-alt': { fileSize: 40, type: 'file' },
'service2/Dockerfile-alt': { fileSize: 13, type: 'file' },
'service2/.dockerignore': { fileSize: 12, type: 'file' },
'service2/file2-crlf.sh': {
fileSize: isWindows ? 12 : 14,

2
tests/docker-build.ts
View File

@ -198,7 +198,7 @@ export async function testDockerBuildStream(o: {
tag,
});
if (o.commandLine.startsWith('build')) {
o.dockerMock.expectGetImages();
o.dockerMock.expectGetImages({ optional: true });
}
}

38
tests/nock/docker-mock.ts
View File

@ -133,4 +133,42 @@ export class DockerMock extends NockMock {
},
);
}
public expectGetManifestBusybox(opts: ScopeOpts = {}) {
// this.optGet(/^\/distribution\/.*/, opts).replyWithFile(
this.optGet('/distribution/busybox/json', opts).replyWithFile(
200,
path.join(dockerResponsePath, 'distribution-busybox-GET.json'),
{
'api-version': '1.38',
'Content-Type': 'application/json',
},
);
}
public expectGetManifestRpi3Alpine(opts: ScopeOpts = {}) {
this.optGet(
'/distribution/balenalib/raspberrypi3-alpine/json',
opts,
).replyWithFile(
200,
path.join(dockerResponsePath, 'distribution-rpi3alpine.json'),
{
'api-version': '1.38',
'Content-Type': 'application/json',
},
);
}
public expectGetManifestNucAlpine(opts: ScopeOpts = {}) {
// NOTE: This URL does no work in real life... it's "intel-nuc", not "nuc"
this.optGet('/distribution/balenalib/nuc-alpine/json', opts).replyWithFile(
200,
path.join(dockerResponsePath, 'distribution-nucalpine.json'),
{
'api-version': '1.38',
'Content-Type': 'application/json',
},
);
}
}

7
tests/nock/nock-mock.ts
View File

@ -33,7 +33,10 @@ export class NockMock {
public readonly expect;
protected static instanceCount = 0;
constructor(public basePathPattern: string | RegExp) {
constructor(
public basePathPattern: string | RegExp,
public allowUnmocked: boolean = false,
) {
if (NockMock.instanceCount === 0) {
if (!nock.isActive()) {
nock.activate();
@ -45,7 +48,7 @@ export class NockMock {
);
}
NockMock.instanceCount += 1;
this.scope = nock(this.basePathPattern);
this.scope = nock(this.basePathPattern, { allowUnmocked });
this.expect = this.scope;
}

53
tests/test-data/docker-response/distribution-busybox-GET.json Normal file
View File

@ -0,0 +1,53 @@
{
"Descriptor": {
"mediaType": "application/vnd.docker.distribution.manifest.list.v2+json",
"digest": "sha256:52f73a0a43a16cf37cd0720c90887ce972fe60ee06a687ee71fb93a7ca601df7",
"size": 2295
},
"Platforms": [
{
"architecture": "amd64",
"os": "linux"
},
{
"architecture": "arm",
"os": "linux",
"variant": "v5"
},
{
"architecture": "arm",
"os": "linux",
"variant": "v6"
},
{
"architecture": "arm",
"os": "linux",
"variant": "v7"
},
{
"architecture": "arm64",
"os": "linux",
"variant": "v8"
},
{
"architecture": "386",
"os": "linux"
},
{
"architecture": "mips64le",
"os": "linux"
},
{
"architecture": "ppc64le",
"os": "linux"
},
{
"architecture": "riscv64",
"os": "linux"
},
{
"architecture": "s390x",
"os": "linux"
}
]
}

13
tests/test-data/docker-response/distribution-nucalpine.json Normal file
View File

@ -0,0 +1,13 @@
{
"Descriptor": {
"mediaType": "application/vnd.docker.distribution.manifest.v2+json",
"digest": "sha256:d70bb0dd863198b41ea5d638993a9fbb912b3ea54b36480d1dc13e6b5b29021a",
"size": 2610
},
"Platforms": [
{
"architecture": "amd64",
"os": "linux"
}
]
}

14
tests/test-data/docker-response/distribution-rpi3alpine.json Normal file
View File

@ -0,0 +1,14 @@
{
"Descriptor": {
"mediaType": "application/vnd.docker.distribution.manifest.v2+json",
"digest": "sha256:2e33dc19d8514e01f7676532c507ddd95d0be20497fee25f4cbfc972cc6343d0",
"size": 2821
},
"Platforms": [
{
"architecture": "arm",
"os": "linux",
"variant": "v7"
}
]
}

2
tests/test-data/projects/docker-compose/basic/service2/Dockerfile-alt
View File

@ -1 +1 @@
alternative Dockerfile (basic/service2)
FROM busybox