Switch from Bluebird.map to native version

Change-type: patch
This commit is contained in:
Pagan Gazzard 2020-06-30 21:17:24 +01:00
parent 41a8e1b2a4
commit d4f288394b
6 changed files with 206 additions and 179 deletions

View File

@ -103,15 +103,17 @@ async function buildPkg() {
['*', ['opn', 'xdg-open'], ['xdg-open-402']],
['darwin', ['denymount', 'bin', 'denymount'], ['denymount']],
];
await Bluebird.map(paths, ([platform, source, dest]) => {
if (platform === '*' || platform === process.platform) {
// eg copy from node_modules/open/xdg-open to build-bin/xdg-open
return fs.copy(
path.join(ROOT, 'node_modules', ...source),
path.join(ROOT, 'build-bin', ...dest),
);
}
});
await Promise.all(
paths.map(([platform, source, dest]) => {
if (platform === '*' || platform === process.platform) {
// eg copy from node_modules/open/xdg-open to build-bin/xdg-open
return fs.copy(
path.join(ROOT, 'node_modules', ...source),
path.join(ROOT, 'build-bin', ...dest),
);
}
}),
);
const nativeExtensionPaths: string[] = await filehound
.create()
.paths(path.join(ROOT, 'node_modules'))
@ -120,12 +122,14 @@ async function buildPkg() {
console.log(`\nCopying to build-bin:\n${nativeExtensionPaths.join('\n')}`);
await Bluebird.map(nativeExtensionPaths, (extPath) =>
fs.copy(
extPath,
extPath.replace(
path.join(ROOT, 'node_modules'),
path.join(ROOT, 'build-bin'),
await Promise.all(
nativeExtensionPaths.map((extPath) =>
fs.copy(
extPath,
extPath.replace(
path.join(ROOT, 'node_modules'),
path.join(ROOT, 'build-bin'),
),
),
),
);

View File

@ -263,9 +263,8 @@ export default class OsConfigureCmd extends Command {
);
if (options['system-connection']) {
const files = await Bluebird.map(
options['system-connection'],
async (filePath) => {
const files = await Promise.all(
options['system-connection'].map(async (filePath) => {
const content = await fs.readFile(filePath, 'utf8');
const name = path.basename(filePath);
@ -273,7 +272,7 @@ export default class OsConfigureCmd extends Command {
name,
content,
};
},
}),
);
await Bluebird.each(files, async ({ name, content }) => {

View File

@ -105,9 +105,8 @@ export default class ScanCmd extends Command {
}
// Query devices for info
const devicesInfo = await Bluebird.map(
activeLocalDevices,
({ host, address }) => {
const devicesInfo = await Promise.all(
activeLocalDevices.map(({ host, address }) => {
const docker = dockerUtils.createClient({
host: address,
port: dockerPort,
@ -123,7 +122,7 @@ export default class ScanCmd extends Command {
.versionAsync()
.catchReturn('Could not get Docker version'),
});
},
}),
);
// Reduce properties if not --verbose

View File

@ -367,13 +367,15 @@ export function buildProject(
}
logger.logInfo('Emulation is enabled');
// Copy qemu into all build contexts
return Bluebird.map(imageDescriptors, function (d) {
if (typeof d.image === 'string' || d.image.context == null) {
return;
}
// external image
return qemu.copyQemu(path.join(projectPath, d.image.context), arch);
});
return Promise.all(
imageDescriptors.map(function (d) {
if (typeof d.image === 'string' || d.image.context == null) {
return;
}
// external image
return qemu.copyQemu(path.join(projectPath, d.image.context), arch);
}),
);
})
.then((
needsQemu, // Tar up the directory, ready for the build stream
@ -488,57 +490,64 @@ export function buildProject(
})
.then(function (tasks) {
logger.logDebug('Prepared tasks; building...');
return Bluebird.map(
builder.performBuilds(tasks, docker, BALENA_ENGINE_TMP_PATH),
function (builtImage) {
if (!builtImage.successful) {
/** @type {Error & {serviceName?: string}} */
const error = builtImage.error ?? new Error();
error.serviceName = builtImage.serviceName;
throw error;
}
return builder
.performBuilds(tasks, docker, BALENA_ENGINE_TMP_PATH)
.then(function (builtImages) {
return Promise.all(
builtImages.map(function (builtImage) {
if (!builtImage.successful) {
/** @type {Error & {serviceName?: string}} */
const error = builtImage.error ?? new Error();
error.serviceName = builtImage.serviceName;
throw error;
}
const d = imageDescriptorsByServiceName[builtImage.serviceName];
const task = _.find(tasks, { serviceName: builtImage.serviceName });
const d = imageDescriptorsByServiceName[builtImage.serviceName];
const task = _.find(tasks, {
serviceName: builtImage.serviceName,
});
const image = {
serviceName: d.serviceName,
name: typeof d.image === 'string' ? d.image : d.image.tag,
logs: truncateString(task.logBuffer.join('\n'), LOG_LENGTH_MAX),
props: {
dockerfile: builtImage.dockerfile,
projectType: builtImage.projectType,
},
};
const image = {
serviceName: d.serviceName,
name: typeof d.image === 'string' ? d.image : d.image.tag,
logs: truncateString(task.logBuffer.join('\n'), LOG_LENGTH_MAX),
props: {
dockerfile: builtImage.dockerfile,
projectType: builtImage.projectType,
},
};
// Times here are timestamps, so test whether they're null
// before creating a date out of them, as `new Date(null)`
// creates a date representing UNIX time 0.
if (builtImage.startTime) {
image.props.startTime = new Date(builtImage.startTime);
}
if (builtImage.endTime) {
image.props.endTime = new Date(builtImage.endTime);
}
return docker
.getImage(image.name)
.inspect()
.get('Size')
.then((size) => {
image.props.size = size;
})
.return(image);
},
).tap(function (images) {
const summary = _(images)
.map(({ serviceName, props }) => [
serviceName,
`Image size: ${humanize.filesize(props.size)}`,
])
.fromPairs()
.value();
renderer.end(summary);
});
// Times here are timestamps, so test whether they're null
// before creating a date out of them, as `new Date(null)`
// creates a date representing UNIX time 0.
if (builtImage.startTime) {
image.props.startTime = new Date(builtImage.startTime);
}
if (builtImage.endTime) {
image.props.endTime = new Date(builtImage.endTime);
}
return docker
.getImage(image.name)
.inspect()
.get('Size')
.then((size) => {
image.props.size = size;
})
.return(image);
}),
);
})
.then(function (images) {
const summary = _(images)
.map(({ serviceName, props }) => [
serviceName,
`Image size: ${humanize.filesize(props.size)}`,
])
.fromPairs()
.value();
renderer.end(summary);
return images;
});
})
.finally(renderer.end);
}
@ -598,32 +607,34 @@ export const createRelease = function (
* @param {import('docker-toolbelt')} docker
* @param {Array<import('./compose-types').BuiltImage>} images
* @param {Partial<import('balena-release/build/models').ImageModel>} serviceImages
* @returns {Bluebird<Array<import('./compose-types').TaggedImage>>}
* @returns {Promise<Array<import('./compose-types').TaggedImage>>}
*/
export const tagServiceImages = (docker, images, serviceImages) =>
Bluebird.map(images, function (d) {
const serviceImage = serviceImages[d.serviceName];
const imageName = serviceImage.is_stored_at__image_location;
const match = /(.*?)\/(.*?)(?::([^/]*))?$/.exec(imageName);
if (match == null) {
throw new Error(`Could not parse imageName: '${imageName}'`);
}
const [, registry, repo, tag = 'latest'] = match;
const name = `${registry}/${repo}`;
return docker
.getImage(d.name)
.tag({ repo: name, tag, force: true })
.then(() => docker.getImage(`${name}:${tag}`))
.then((localImage) => ({
serviceName: d.serviceName,
serviceImage,
localImage,
registry,
repo,
logs: d.logs,
props: d.props,
}));
});
Promise.all(
images.map(function (d) {
const serviceImage = serviceImages[d.serviceName];
const imageName = serviceImage.is_stored_at__image_location;
const match = /(.*?)\/(.*?)(?::([^/]*))?$/.exec(imageName);
if (match == null) {
throw new Error(`Could not parse imageName: '${imageName}'`);
}
const [, registry, repo, tag = 'latest'] = match;
const name = `${registry}/${repo}`;
return docker
.getImage(d.name)
.tag({ repo: name, tag, force: true })
.then(() => docker.getImage(`${name}:${tag}`))
.then((localImage) => ({
serviceName: d.serviceName,
serviceImage,
localImage,
registry,
repo,
logs: d.logs,
props: d.props,
}));
}),
);
/**
* @param {*} sdk
@ -655,15 +666,19 @@ export const getPreviousRepos = (sdk, docker, logger, appID) =>
// grab all images from the latest release, return all image locations in the registry
if (release.length > 0) {
const images = release[0].contains__image;
return Bluebird.map(images, function (d) {
const imageName = d.image[0].is_stored_at__image_location;
return docker.getRegistryAndName(imageName).then(function (registry) {
logger.logDebug(
`Requesting access to previously pushed image repo (${registry.imageName})`,
);
return registry.imageName;
});
});
return Promise.all(
images.map(function (d) {
const imageName = d.image[0].is_stored_at__image_location;
return docker
.getRegistryAndName(imageName)
.then(function (registry) {
logger.logDebug(
`Requesting access to previously pushed image repo (${registry.imageName})`,
);
return registry.imageName;
});
}),
);
} else {
return [];
}
@ -733,41 +748,43 @@ export const pushAndUpdateServiceImages = function (
const reporters = progress.aggregateProgress(images.length, renderer);
return Bluebird.using(tty.cursorHidden(), () =>
Bluebird.map(images, ({ serviceImage, localImage, props, logs }, index) =>
Bluebird.join(
// @ts-ignore
localImage.inspect().get('Size'),
retry(
Promise.all(
images.map(({ serviceImage, localImage, props, logs }, index) =>
Bluebird.join(
// @ts-ignore
() => progress.push(localImage.name, reporters[index], opts),
3, // `times` - retry 3 times
// @ts-ignore
localImage.name, // `label` included in retry log messages
2000, // `delayMs` - wait 2 seconds before the 1st retry
1.4, // `backoffScaler` - wait multiplier for each retry
).finally(renderer.end),
/** @type {(size: number, digest: string) => void} */
function (size, digest) {
serviceImage.image_size = size;
serviceImage.content_hash = digest;
serviceImage.build_log = logs;
serviceImage.dockerfile = props.dockerfile;
serviceImage.project_type = props.projectType;
if (props.startTime) {
serviceImage.start_timestamp = props.startTime;
}
if (props.endTime) {
serviceImage.end_timestamp = props.endTime;
}
serviceImage.push_timestamp = new Date();
serviceImage.status = 'success';
},
)
.tapCatch(function (e) {
serviceImage.error_message = '' + e;
serviceImage.status = 'failed';
})
.finally(() => afterEach?.(serviceImage, props)),
localImage.inspect().get('Size'),
retry(
// @ts-ignore
() => progress.push(localImage.name, reporters[index], opts),
3, // `times` - retry 3 times
// @ts-ignore
localImage.name, // `label` included in retry log messages
2000, // `delayMs` - wait 2 seconds before the 1st retry
1.4, // `backoffScaler` - wait multiplier for each retry
).finally(renderer.end),
/** @type {(size: number, digest: string) => void} */
function (size, digest) {
serviceImage.image_size = size;
serviceImage.content_hash = digest;
serviceImage.build_log = logs;
serviceImage.dockerfile = props.dockerfile;
serviceImage.project_type = props.projectType;
if (props.startTime) {
serviceImage.start_timestamp = props.startTime;
}
if (props.endTime) {
serviceImage.end_timestamp = props.endTime;
}
serviceImage.push_timestamp = new Date();
serviceImage.status = 'success';
},
)
.tapCatch(function (e) {
serviceImage.error_message = '' + e;
serviceImage.status = 'failed';
})
.finally(() => afterEach?.(serviceImage, props)),
),
),
);
};

View File

@ -843,7 +843,9 @@ export async function deployProject(
throw err;
} finally {
logger.logDebug('Untagging images...');
await Bluebird.map(taggedImages, ({ localImage }) => localImage.remove());
await Promise.all(
taggedImages.map(({ localImage }) => localImage.remove()),
);
}
} finally {
runloop = runSpinner(tty, spinner, `${prefix}Saving release...`);

View File

@ -361,20 +361,24 @@ export async function performBuilds(
// Now tag any external images with the correct name that they should be,
// as this won't be done by resin-multibuild
await Bluebird.map(localImages, async (localImage) => {
if (localImage.external) {
// We can be sure that localImage.name is set here, because of the failure code above
const image = docker.getImage(localImage.name!);
await image.tag({
repo: generateImageName(localImage.serviceName),
force: true,
});
imagesToRemove.push(localImage.name!);
}
});
await Promise.all(
localImages.map(async (localImage) => {
if (localImage.external) {
// We can be sure that localImage.name is set here, because of the failure code above
const image = docker.getImage(localImage.name!);
await image.tag({
repo: generateImageName(localImage.serviceName),
force: true,
});
imagesToRemove.push(localImage.name!);
}
}),
);
await Bluebird.map(_.uniq(imagesToRemove), (image) =>
docker.getImage(image).remove({ force: true }),
await Promise.all(
_.uniq(imagesToRemove).map((image) =>
docker.getImage(image).remove({ force: true }),
),
);
return buildTasks;
@ -512,26 +516,28 @@ async function assignDockerBuildOpts(
globalLogger.logDebug(`Using ${images.length} on-device images for cache...`);
await Bluebird.map(buildTasks, async (task: BuildTask) => {
task.dockerOpts = {
cachefrom: images,
labels: {
'io.resin.local.image': '1',
'io.resin.local.service': task.serviceName,
},
t: generateImageName(task.serviceName),
nocache: opts.nocache,
forcerm: true,
};
if (task.external) {
task.dockerOpts.authconfig = await getAuthConfigObj(
task.imageName!,
opts.registrySecrets,
);
} else {
task.dockerOpts.registryconfig = opts.registrySecrets;
}
});
await Promise.all(
buildTasks.map(async (task: BuildTask) => {
task.dockerOpts = {
cachefrom: images,
labels: {
'io.resin.local.image': '1',
'io.resin.local.service': task.serviceName,
},
t: generateImageName(task.serviceName),
nocache: opts.nocache,
forcerm: true,
};
if (task.external) {
task.dockerOpts.authconfig = await getAuthConfigObj(
task.imageName!,
opts.registrySecrets,
);
} else {
task.dockerOpts.registryconfig = opts.registrySecrets;
}
}),
);
}
function generateImageName(serviceName: string): string {