Switch to balena-compose

Removes a bunch of individual dependencies by switching to `@balena/compose` which (currently) groups and manages those dependencies together in one package.

Change-type: minor
This commit is contained in:
Akis Kesoglou 2022-06-28 12:22:42 +00:00
parent 21ded85c7a
commit c7f56d92dd
20 changed files with 310 additions and 783 deletions

View File

@ -16,7 +16,7 @@
*/
import { flags } from '@oclif/command';
import type { ImageDescriptor } from 'resin-compose-parse';
import type { ImageDescriptor } from '@balena/compose/dist/parse';
import Command from '../command';
import { ExpectedError } from '../errors';

View File

@ -22,7 +22,7 @@ import { getBalenaSdk, stripIndent } from '../utils/lazy';
import { dockerignoreHelp, registrySecretsHelp } from '../utils/messages';
import type { BalenaSDK } from 'balena-sdk';
import { ExpectedError, instanceOf } from '../errors';
import { RegistrySecrets } from 'resin-multibuild';
import { RegistrySecrets } from '@balena/compose/dist/multibuild';
import { lowercaseIfSlug } from '../utils/normalization';
import {
applyReleaseTagKeysAndValues,

View File

@ -15,8 +15,11 @@
* limitations under the License.
*/
import type { ImageModel, ReleaseModel } from 'balena-release/build/models';
import type { Composition, ImageDescriptor } from 'resin-compose-parse';
import type {
ImageModel,
ReleaseModel,
} from '@balena/compose/dist/release/models';
import type { Composition, ImageDescriptor } from '@balena/compose/dist/parse';
import type { Pack } from 'tar-stream';
interface Image {
@ -39,7 +42,7 @@ export interface BuiltImage {
export interface TaggedImage {
localImage: import('dockerode').Image;
serviceImage: import('balena-release/build/models').ImageModel;
serviceImage: import('@balena/compose/dist/release/models').ImageModel;
serviceName: string;
logs: string;
props: BuiltImage.props;
@ -78,7 +81,9 @@ export interface ComposeProject {
}
export interface Release {
client: ReturnType<typeof import('balena-release').createClient>;
client: ReturnType<
typeof import('@balena/compose/dist/release').createClient
>;
release: Pick<
ReleaseModel,
| 'id'

View File

@ -19,7 +19,7 @@ import type { Renderer } from './compose_ts';
import type * as SDK from 'balena-sdk';
import type Dockerode = require('dockerode');
import * as path from 'path';
import type { Composition, ImageDescriptor } from 'resin-compose-parse';
import type { Composition, ImageDescriptor } from '@balena/compose/dist/parse';
import type {
BuiltImage,
ComposeOpts,
@ -64,7 +64,7 @@ export function createProject(
): ComposeProject {
const yml = require('js-yaml') as typeof import('js-yaml');
const compose =
require('resin-compose-parse') as typeof import('resin-compose-parse');
require('@balena/compose/dist/parse') as typeof import('@balena/compose/dist/parse');
// both methods below may throw.
const rawComposition = yml.load(composeStr);
@ -107,7 +107,7 @@ export const createRelease = async function (
const _ = require('lodash') as typeof import('lodash');
const crypto = require('crypto') as typeof import('crypto');
const releaseMod =
require('balena-release') as typeof import('balena-release');
require('@balena/compose/dist/release') as typeof import('@balena/compose/dist/release');
const client = releaseMod.createClient({ apiEndpoint, auth });
@ -214,7 +214,7 @@ export const getPreviousRepos = (
image: [SDK.Image];
}>;
const { getRegistryAndName } =
require('resin-multibuild') as typeof import('resin-multibuild');
require('@balena/compose/dist/multibuild') as typeof import('@balena/compose/dist/multibuild');
return Promise.all(
images.map(function (d) {
const imageName = d.image[0].is_stored_at__image_location || '';

View File

@ -16,7 +16,7 @@
*/
import { flags } from '@oclif/command';
import { BalenaSDK } from 'balena-sdk';
import type { TransposeOptions } from 'docker-qemu-transpose';
import type { TransposeOptions } from '@balena/compose/dist/emulate';
import type * as Dockerode from 'dockerode';
import { promises as fs } from 'fs';
import jsyaml = require('js-yaml');
@ -26,8 +26,8 @@ import type {
BuildConfig,
Composition,
ImageDescriptor,
} from 'resin-compose-parse';
import type * as MultiBuild from 'resin-multibuild';
} from '@balena/compose/dist/parse';
import type * as MultiBuild from '@balena/compose/dist/multibuild';
import * as semver from 'semver';
import type { Duplex, Readable } from 'stream';
import type { Pack } from 'tar-stream';
@ -118,7 +118,7 @@ export async function loadProject(
image?: string,
imageTag?: string,
): Promise<ComposeProject> {
const compose = await import('resin-compose-parse');
const compose = await import('@balena/compose/dist/parse');
const { createProject } = await import('./compose');
let composeName: string;
let composeStr: string;
@ -262,7 +262,7 @@ export async function buildProject(
opts: BuildProjectOpts,
): Promise<BuiltImage[]> {
await checkBuildSecretsRequirements(opts.docker, opts.projectPath);
const compose = await import('resin-compose-parse');
const compose = await import('@balena/compose/dist/parse');
const imageDescriptors = compose.parse(opts.composition);
const renderer = await startRenderer({ imageDescriptors, ...opts });
let buildSummaryByService: Dictionary<string> | undefined;
@ -333,7 +333,7 @@ async function $buildProject(
logger.logDebug('Prepared tasks; building...');
const { BALENA_ENGINE_TMP_PATH } = await import('../config');
const builder = await import('resin-multibuild');
const builder = await import('@balena/compose/dist/multibuild');
const builtImages = await builder.performBuilds(
tasks,
@ -481,8 +481,9 @@ async function qemuTransposeBuildStream({
throw new Error(`No buildStream for task '${task.tag}'`);
}
const transpose = await import('docker-qemu-transpose');
const { toPosixPath } = (await import('resin-multibuild')).PathUtils;
const transpose = await import('@balena/compose/dist/emulate');
const { toPosixPath } = (await import('@balena/compose/dist/multibuild'))
.PathUtils;
const transposeOptions: TransposeOptions = {
hostQemuPath: toPosixPath(binPath),
@ -508,9 +509,9 @@ async function setTaskProgressHooks({
inlineLogs?: boolean;
renderer: Renderer;
task: BuildTaskPlus;
transposeOptions?: import('docker-qemu-transpose').TransposeOptions;
transposeOptions?: import('@balena/compose/dist/emulate').TransposeOptions;
}) {
const transpose = await import('docker-qemu-transpose');
const transpose = await import('@balena/compose/dist/emulate');
// Get the service-specific log stream
const logStream = renderer.streams[task.serviceName];
task.logBuffer = [];
@ -724,16 +725,16 @@ export async function getServiceDirsFromComposition(
*
* The `image` argument may therefore refer to either a `build` or `image` property
* of a service in a docker-compose.yml file, which is a bit confusing but it matches
* the `ImageDescriptor.image` property as defined by `resin-compose-parse`.
* the `ImageDescriptor.image` property as defined by `@balena/compose/parse`.
*
* Note that `resin-compose-parse` "normalizes" the docker-compose.yml file such
* Note that `@balena/compose/parse` "normalizes" the docker-compose.yml file such
* that, if `services.service.build` is a string, it is converted to a BuildConfig
* object with the string value assigned to `services.service.build.context`:
* https://github.com/balena-io-modules/resin-compose-parse/blob/v2.1.3/src/compose.ts#L166-L167
* https://github.com/balena-io-modules/balena-compose/blob/v0.1.0/lib/parse/compose.ts#L166-L167
* This is why this implementation works when `services.service.build` is defined
* as a string in the docker-compose.yml file.
*
* @param image The `ImageDescriptor.image` attribute parsed with `resin-compose-parse`
* @param image The `ImageDescriptor.image` attribute parsed with `@balena/compose/parse`
*/
export function isBuildConfig(
image: string | BuildConfig,
@ -759,7 +760,8 @@ export async function tarDirectory(
}: TarDirectoryOptions,
): Promise<import('stream').Readable> {
const { filterFilesWithDockerignore } = await import('./ignore');
const { toPosixPath } = (await import('resin-multibuild')).PathUtils;
const { toPosixPath } = (await import('@balena/compose/dist/multibuild'))
.PathUtils;
let readFile: (file: string) => Promise<Buffer>;
if (process.platform === 'win32') {
@ -941,7 +943,7 @@ async function parseRegistrySecrets(
throw new ExpectedError('Filename must end with .json, .yml or .yaml');
}
const raw = (await fs.readFile(secretsFilename)).toString();
const multiBuild = await import('resin-multibuild');
const multiBuild = await import('@balena/compose/dist/multibuild');
const registrySecrets =
new multiBuild.RegistrySecretValidator().validateRegistrySecrets(
isYaml ? require('js-yaml').load(raw) : JSON.parse(raw),
@ -970,7 +972,7 @@ export async function makeBuildTasks(
releaseHash: string = 'unavailable',
preprocessHook?: (dockerfile: string) => string,
): Promise<MultiBuild.BuildTask[]> {
const multiBuild = await import('resin-multibuild');
const multiBuild = await import('@balena/compose/dist/multibuild');
const buildTasks = await multiBuild.splitBuildStream(composition, tarStream);
logger.logDebug('Found build tasks:');
@ -1016,7 +1018,7 @@ async function performResolution(
releaseHash: string,
preprocessHook?: (dockerfile: string) => string,
): Promise<MultiBuild.BuildTask[]> {
const multiBuild = await import('resin-multibuild');
const multiBuild = await import('@balena/compose/dist/multibuild');
const resolveListeners: MultiBuild.ResolveListeners = {};
const resolvePromise = new Promise<never>((_resolve, reject) => {
resolveListeners.error = [reject];
@ -1081,7 +1083,7 @@ async function validateSpecifiedDockerfile(
dockerfilePath: string,
): Promise<string> {
const { contains, toNativePath, toPosixPath } = (
await import('resin-multibuild')
await import('@balena/compose/dist/multibuild')
).PathUtils;
const nativeProjectPath = path.normalize(projectPath);
@ -1241,7 +1243,7 @@ async function pushAndUpdateServiceImages(
token: string,
images: TaggedImage[],
afterEach: (
serviceImage: import('balena-release/build/models').ImageModel,
serviceImage: import('@balena/compose/dist/release/models').ImageModel,
props: object,
) => void,
) {
@ -1326,12 +1328,14 @@ async function pushAndUpdateServiceImages(
async function pushServiceImages(
docker: Dockerode,
logger: Logger,
pineClient: ReturnType<typeof import('balena-release').createClient>,
pineClient: ReturnType<
typeof import('@balena/compose/dist/release').createClient
>,
taggedImages: TaggedImage[],
token: string,
skipLogUpload: boolean,
): Promise<void> {
const releaseMod = await import('balena-release');
const releaseMod = await import('@balena/compose/dist/release');
logger.logInfo('Pushing images to registry...');
await pushAndUpdateServiceImages(
docker,
@ -1361,8 +1365,8 @@ export async function deployProject(
skipLogUpload: boolean,
projectPath: string,
isDraft: boolean,
): Promise<import('balena-release/build/models').ReleaseModel> {
const releaseMod = await import('balena-release');
): Promise<import('@balena/compose/dist/release/models').ReleaseModel> {
const releaseMod = await import('@balena/compose/dist/release');
const { createRelease, tagServiceImages } = await import('./compose');
const tty = (await import('./tty'))(process.stdout);

View File

@ -18,13 +18,13 @@
import * as semver from 'balena-semver';
import * as Docker from 'dockerode';
import * as _ from 'lodash';
import { Composition } from 'resin-compose-parse';
import { Composition } from '@balena/compose/dist/parse';
import {
BuildTask,
getAuthConfigObj,
LocalImage,
RegistrySecrets,
} from 'resin-multibuild';
} from '@balena/compose/dist/multibuild';
import type { Readable } from 'stream';
import { BALENA_ENGINE_TMP_PATH } from '../../config';
@ -321,7 +321,7 @@ async function performBuilds(
opts: DeviceDeployOptions,
buildLogs?: Dictionary<string>,
): Promise<BuildTask[]> {
const multibuild = await import('resin-multibuild');
const multibuild = await import('@balena/compose/dist/multibuild');
const buildTasks = await makeBuildTasks(
composition,
@ -370,7 +370,7 @@ async function performBuilds(
const imagesToRemove: string[] = [];
// Now tag any external images with the correct name that they should be,
// as this won't be done by resin-multibuild
// as this won't be done by @balena/compose/multibuild
await Promise.all(
localImages.map(async (localImage) => {
if (localImage.external) {
@ -414,7 +414,7 @@ export async function rebuildSingleTask(
// this should provide the following callback
containerIdCb?: (id: string) => void,
): Promise<string> {
const multibuild = await import('resin-multibuild');
const multibuild = await import('@balena/compose/dist/multibuild');
// First we run the build task, to get the new image id
let buildLogs = '';
const logHandler = (_s: string, line: string) => {
@ -533,10 +533,17 @@ async function assignDockerBuildOpts(
await Promise.all(
buildTasks.map(async (task: BuildTask) => {
task.dockerOpts = {
cachefrom: images,
labels: {
'io.resin.local.image': '1',
'io.resin.local.service': task.serviceName,
...(task.dockerOpts || {}),
...{
cachefrom: images,
labels: {
'io.resin.local.image': '1',
'io.resin.local.service': task.serviceName,
},
t: getImageNameFromTask(task),
nocache: opts.nocache,
forcerm: true,
pull: opts.pull,
},
t: getImageNameFromTask(task),
nocache: opts.nocache,

View File

@ -21,8 +21,8 @@ import * as fs from 'fs';
import Livepush, { ContainerNotRunningError } from 'livepush';
import * as _ from 'lodash';
import * as path from 'path';
import type { Composition } from 'resin-compose-parse';
import type { BuildTask } from 'resin-multibuild';
import type { Composition } from '@balena/compose/dist/parse';
import type { BuildTask } from '@balena/compose/dist/multibuild';
import { instanceOf } from '../../errors';
import Logger = require('../logger');

View File

@ -105,7 +105,7 @@ export interface BuildOpts {
cachefrom?: string[];
nocache?: boolean;
pull?: boolean;
registryconfig?: import('resin-multibuild').RegistrySecrets;
registryconfig?: import('@balena/compose/dist/multibuild').RegistrySecrets;
squash?: boolean;
t?: string; // only the tag portion of the image name, e.g. 'abc' in 'myimg:abc'
}
@ -132,7 +132,7 @@ export function generateBuildOpts(options: {
'cache-from'?: string;
nocache: boolean;
pull?: boolean;
'registry-secrets'?: import('resin-multibuild').RegistrySecrets;
'registry-secrets'?: import('@balena/compose/dist/multibuild').RegistrySecrets;
squash: boolean;
tag?: string;
}): BuildOpts {

View File

@ -17,7 +17,7 @@ import type { BalenaSDK } from 'balena-sdk';
import * as JSONStream from 'JSONStream';
import * as readline from 'readline';
import * as request from 'request';
import { RegistrySecrets } from 'resin-multibuild';
import { RegistrySecrets } from '@balena/compose/dist/multibuild';
import type * as Stream from 'stream';
import streamToPromise = require('stream-to-promise');
import type { Pack } from 'tar-stream';

906
npm-shrinkwrap.json generated

File diff suppressed because it is too large Load Diff

View File

@ -29,7 +29,7 @@
"node_modules/balena-sdk/es2018/index.js",
"node_modules/balena-sync/build/**/*.js",
"node_modules/pinejs-client-request/node_modules/pinejs-client-core/es2018/index.js",
"node_modules/resin-compose-parse/build/schemas/*.json"
"node_modules/@balena/compose/dist/parse/schemas/*.json"
],
"assets": [
"build/auth/pages/*.ejs",
@ -194,6 +194,7 @@
"typescript": "^4.6.4"
},
"dependencies": {
"@balena/compose": "^2.1.0",
"@balena/dockerignore": "^1.0.2",
"@balena/es-version": "^1.0.1",
"@oclif/command": "^1.8.16",
@ -208,7 +209,6 @@
"balena-image-fs": "^7.0.6",
"balena-image-manager": "^8.0.0",
"balena-preload": "^12.1.0",
"balena-release": "^3.2.0",
"balena-sdk": "^16.22.0",
"balena-semver": "^2.3.0",
"balena-settings-client": "^4.0.7",
@ -226,7 +226,6 @@
"denymount": "^2.3.0",
"docker-modem": "3.0.0",
"docker-progress": "^5.1.3",
"docker-qemu-transpose": "^1.1.1",
"dockerode": "^3.3.1",
"ejs": "^3.1.6",
"etcher-sdk": "^6.2.1",
@ -264,9 +263,7 @@
"request": "^2.88.2",
"resin-cli-form": "^2.0.2",
"resin-cli-visuals": "^1.8.0",
"resin-compose-parse": "^2.1.3",
"resin-doodles": "^0.2.0",
"resin-multibuild": "^4.12.2",
"resin-stream-logger": "^0.1.2",
"rimraf": "^3.0.2",
"semver": "^7.3.5",

View File

@ -14,7 +14,7 @@ upstream:
url: 'https://github.com/balena-io-modules/balena-sync'
- repo: 'etcher-sdk'
url: 'https://github.com/balena-io-modules/etcher-sdk/'
- repo: 'resin-compose-parse'
url: 'https://github.com/balena-io-modules/resin-compose-parse'
- repo: 'balena-compose'
url: 'https://github.com/balena-io-modules/balena-compose'
- repo: 'docker-progress'
url: 'https://github.com/balena-io-modules/docker-progress'

View File

@ -15,7 +15,7 @@
* limitations under the License.
*/
import type { Request as ReleaseRequest } from 'balena-release';
import type { Request as ReleaseRequest } from '@balena/compose/dist/release';
import { expect } from 'chai';
import { promises as fs } from 'fs';
import * as _ from 'lodash';
@ -291,7 +291,7 @@ describe('balena deploy', function () {
statusCode: 500,
inspectRequest: (_uri, requestBody) => {
const imageBody = requestBody as Partial<
import('balena-release/build/models').ImageModel
import('@balena/compose/dist/release/models').ImageModel
>;
expect(imageBody.status).to.equal('success');
},
@ -300,7 +300,7 @@ describe('balena deploy', function () {
api.expectPatchRelease({
inspectRequest: (_uri, requestBody) => {
const releaseBody = requestBody as Partial<
import('balena-release/build/models').ReleaseModel
import('@balena/compose/dist/release/models').ReleaseModel
>;
expect(releaseBody.status).to.equal('failed');
},

View File

@ -19,7 +19,7 @@ import { expect } from 'chai';
import * as _ from 'lodash';
import { promises as fs } from 'fs';
import * as path from 'path';
import { PathUtils } from 'resin-multibuild';
import { PathUtils } from '@balena/compose/dist/multibuild';
import rewire = require('rewire');
import * as sinon from 'sinon';
import { Readable } from 'stream';

View File

@ -64,6 +64,12 @@
The file must be distributed with executable as %2.
%1: node_modules/patch-package/node_modules/open/xdg-open
%2: path-to-executable/xdg-open
> Warning Cannot resolve 'path'
node_modules/@balena/compose/dist/parse/schemas/index.js
Dynamic require may fail at run time, because the requested file
is unknown at compilation time and not included into executable.
Use a string literal as an argument for 'require', or leave it
as is and specify the resolved file name in 'scripts' option.
> Warning Cannot include file %1 into executable.
The file must be distributed with executable as %2.
%1: node_modules/drivelist/build/Release/drivelist.node

View File

@ -64,6 +64,12 @@
The file must be distributed with executable as %2.
%1: node_modules/patch-package/node_modules/open/xdg-open
%2: path-to-executable/xdg-open
> Warning Cannot resolve 'path'
node_modules/@balena/compose/dist/parse/schemas/index.js
Dynamic require may fail at run time, because the requested file
is unknown at compilation time and not included into executable.
Use a string literal as an argument for 'require', or leave it
as is and specify the resolved file name in 'scripts' option.
> Warning Cannot include file %1 into executable.
The file must be distributed with executable as %2.
%1: node_modules/drivelist/build/Release/drivelist.node

View File

@ -64,6 +64,12 @@
The file must be distributed with executable as %2.
%1: node_modules\patch-package\node_modules\open\xdg-open
%2: path-to-executable/xdg-open
> Warning Cannot resolve 'path'
node_modules\@balena\compose\dist\parse\schemas\index.js
Dynamic require may fail at run time, because the requested file
is unknown at compilation time and not included into executable.
Use a string literal as an argument for 'require', or leave it
as is and specify the resolved file name in 'scripts' option.
> Warning Cannot include file %1 into executable.
The file must be distributed with executable as %2.
%1: node_modules\drivelist\build\Release\drivelist.node

View File

@ -79,7 +79,7 @@ describeSS('LivepushManager::setupFilesystemWatcher', function () {
async function createMonitors(
projectPath: string,
composition: import('resin-compose-parse').Composition,
composition: import('@balena/compose/dist/parse').Composition,
multiDockerignore: boolean,
changedPathHandler: (serviceName: string, changedPath: string) => void,
): Promise<ByService<chokidar.FSWatcher>> {

View File

@ -17,9 +17,11 @@
import { expect } from 'chai';
describe('resin-multibuild consistency', function () {
describe('@balena/compose/multibuild consistency', function () {
it('should use the same values for selected constants', async () => {
const { QEMU_BIN_NAME: MQEMU_BIN_NAME } = await import('resin-multibuild');
const { QEMU_BIN_NAME: MQEMU_BIN_NAME } = await import(
'@balena/compose/dist/multibuild'
);
const { QEMU_BIN_NAME } = await import('../../build/utils/qemu');
expect(QEMU_BIN_NAME).to.equal(MQEMU_BIN_NAME);
});

View File

@ -1,30 +0,0 @@
/**
* @license
* Copyright 2019 Balena Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
declare module 'dockerfile-template' {
/**
* Variables which define what will be replaced, and what they will be replaced with.
*/
export interface TemplateVariables {
[key: string]: string;
}
export function process(
content: string,
variables: TemplateVariables,
): string;
}