Avoid synchronous file accesses for os release info

This also required refactoring the request library to be generated with
a promise, as we now no longer get the information synchronously.

We also cache the release info, to avoid grabbing it again within the
same runtime, which does not make sense.

Change-type: patch
Signed-off-by: Cameron Diver <cameron@balena.io>
This commit is contained in:
Cameron Diver 2019-08-22 13:41:14 +01:00
parent 487e2c54a1
commit 2d168784b2
5 changed files with 118 additions and 77 deletions

View File

@ -21,7 +21,7 @@ import {
InternalInconsistencyError,
} from './lib/errors';
import { pathExistsOnHost } from './lib/fs-utils';
import { request, requestOpts } from './lib/request';
import * as request from './lib/request';
import { writeLock } from './lib/update-lock';
import { DeviceApplicationState } from './types/state';
@ -146,7 +146,7 @@ export class APIBinder {
}
const baseUrl = url.resolve(apiEndpoint, '/v5/');
const passthrough = _.cloneDeep(requestOpts);
const passthrough = _.cloneDeep(await request.getRequestOptions());
passthrough.headers =
passthrough.headers != null ? passthrough.headers : {};
passthrough.headers.Authorization = `Bearer ${currentApiKey}`;
@ -781,7 +781,7 @@ export class APIBinder {
}
// We found the device so we can try to register a working device key for it
const [res] = await request
const [res] = await (await request.getRequestInstance())
.postAsync(`${opts.apiEndpoint}/api-key/device/${device.id}/device-key`, {
json: true,
body: {

View File

@ -13,7 +13,7 @@ import {
ImageAuthenticationError,
InvalidNetGatewayError,
} from './errors';
import { request, requestLib, resumable } from './request';
import * as request from './request';
import { EnvVarObject } from './types';
import log from './supervisor-console';
@ -108,7 +108,7 @@ export class DockerUtils extends DockerToolbelt {
const token = await this.getAuthToken(srcInfo, dstInfo, deltaOpts);
const opts: requestLib.CoreOptions = {
const opts: request.requestLib.CoreOptions = {
followRedirect: false,
timeout: deltaOpts.deltaRequestTimeout,
auth: {
@ -121,7 +121,10 @@ export class DockerUtils extends DockerToolbelt {
deltaOpts.deltaVersion
}/delta?src=${deltaOpts.deltaSource}&dest=${imgDest}`;
const [res, data] = await request.getAsync(url, opts);
const [res, data] = await (await request.getRequestInstance()).getAsync(
url,
opts,
);
if (res.statusCode === 502 || res.statusCode === 504) {
throw new DeltaStillProcessingError();
}
@ -265,7 +268,8 @@ export class DockerUtils extends DockerToolbelt {
): Promise<string> {
logFn('Applying rsync delta...');
return new Promise((resolve, reject) => {
return new Promise(async (resolve, reject) => {
const resumable = await request.getResumableRequest();
const req = resumable(Object.assign({ url: deltaUrl }, opts));
req
.on('progress', onProgress)
@ -328,7 +332,7 @@ export class DockerUtils extends DockerToolbelt {
deltaOpts: DeltaFetchOptions,
): Promise<string> => {
const tokenEndpoint = `${deltaOpts.apiEndpoint}/auth/v1/token`;
const tokenOpts: requestLib.CoreOptions = {
const tokenOpts: request.requestLib.CoreOptions = {
auth: {
user: `d_${deltaOpts.uuid}`,
pass: deltaOpts.currentApiKey,
@ -342,7 +346,7 @@ export class DockerUtils extends DockerToolbelt {
srcInfo.imageName
}:pull`;
const tokenResponseBody = (await request.getAsync(
const tokenResponseBody = (await (await request.getRequestInstance()).getAsync(
tokenUrl,
tokenOpts,
))[1];

View File

@ -1,49 +1,61 @@
import * as Bluebird from 'bluebird';
import * as fs from 'fs';
import * as _ from 'lodash';
import fs = require('mz/fs');
import { InternalInconsistencyError } from './errors';
import log from './supervisor-console';
// FIXME: Don't use synchronous file reading and change call sites to support a promise
function getOSReleaseField(path: string, field: string): string | undefined {
// Retrieve the data for the OS once only per path
const getOSReleaseData = _.memoize(
async (path: string): Promise<Dictionary<string>> => {
const releaseItems: Dictionary<string> = {};
try {
const releaseData = await fs.readFile(path, 'utf-8');
const lines = releaseData.split('\n');
for (const line of lines) {
const [key, ...values] = line.split('=');
// Remove enclosing quotes: http://stackoverflow.com/a/19156197/2549019
const value = _.trim(values.join('=')).replace(/^"(.+(?="$))"$/, '$1');
releaseItems[_.trim(key)] = value;
}
} catch (e) {
throw new InternalInconsistencyError(
`Unable to read file at ${path}: ${e.message} ${e.stack}`,
);
}
return releaseItems;
},
);
async function getOSReleaseField(
path: string,
field: string,
): Promise<string | undefined> {
try {
const releaseData = fs.readFileSync(path, 'utf-8');
const lines = releaseData.split('\n');
const releaseItems: { [field: string]: string } = {};
for (const line of lines) {
const [key, value] = line.split('=');
releaseItems[_.trim(key)] = _.trim(value);
const data = await getOSReleaseData(path);
const value = data[field];
if (value == null) {
log.warn(
`Field ${field} is not available in OS information file: ${path}`,
);
}
if (releaseItems[field] == null) {
throw new Error(`Field ${field} not available in ${path}`);
}
// Remove enclosing quotes: http://stackoverflow.com/a/19156197/2549019
return releaseItems[field].replace(/^"(.+(?="$))"$/, '$1');
} catch (err) {
log.error('Could not get OS release field: ', err);
return;
return data[field];
} catch (e) {
log.warn('Unable to read OS version information: ', e);
}
}
export function getOSVersionSync(path: string): string | undefined {
export async function getOSVersion(path: string): Promise<string | undefined> {
return getOSReleaseField(path, 'PRETTY_NAME');
}
export function getOSVersion(path: string): Bluebird<string | undefined> {
return Bluebird.try(() => {
return getOSVersionSync(path);
});
}
export function getOSVariantSync(path: string): string | undefined {
export function getOSVariant(path: string): Promise<string | undefined> {
return getOSReleaseField(path, 'VARIANT_ID');
}
export function getOSVariant(path: string): Bluebird<string | undefined> {
return Bluebird.try(() => {
return getOSVariantSync(path);
});
export function getOSSemver(path: string): Promise<string | undefined> {
return getOSReleaseField(path, 'VERSION');
}

View File

@ -1,4 +1,5 @@
import * as Bluebird from 'bluebird';
import once = require('lodash/once');
import * as requestLib from 'request';
import * as resumableRequestLib from 'resumable-request';
@ -9,18 +10,6 @@ import supervisorVersion = require('./supervisor-version');
export { requestLib };
const osVersion = osRelease.getOSVersionSync(constants.hostOSVersionPath);
const osVariant = osRelease.getOSVariantSync(constants.hostOSVersionPath);
let userAgent = `Supervisor/${supervisorVersion}`;
if (osVersion != null) {
if (osVariant != null) {
userAgent += ` (Linux; ${osVersion}; ${osVariant})`;
} else {
userAgent += ` (Linux; ${osVersion})`;
}
}
// With these settings, the device must be unable to receive a single byte
// from the network for a continuous period of 20 minutes before we give up.
// (reqTimeout + retryInterval) * retryCount / 1000ms / 60sec ~> minutes
@ -28,20 +17,6 @@ const DEFAULT_REQUEST_TIMEOUT = 30000; // ms
const DEFAULT_REQUEST_RETRY_INTERVAL = 10000; // ms
const DEFAULT_REQUEST_RETRY_COUNT = 30;
export const requestOpts: requestLib.CoreOptions = {
gzip: true,
timeout: DEFAULT_REQUEST_TIMEOUT,
headers: {
'User-Agent': userAgent,
},
};
const resumableOpts = {
timeout: DEFAULT_REQUEST_TIMEOUT,
maxRetries: DEFAULT_REQUEST_RETRY_COUNT,
retryInterval: DEFAULT_REQUEST_RETRY_INTERVAL,
};
type PromisifiedRequest = typeof requestLib & {
postAsync: (
uri: string | requestLib.CoreOptions,
@ -53,9 +28,55 @@ type PromisifiedRequest = typeof requestLib & {
) => Bluebird<any>;
};
const requestHandle = requestLib.defaults(exports.requestOpts);
const getRequestInstances = once(async () => {
// Generate the user agents with out versions
const osVersion = await osRelease.getOSVersion(constants.hostOSVersionPath);
const osVariant = await osRelease.getOSVariant(constants.hostOSVersionPath);
let userAgent = `Supervisor/${supervisorVersion}`;
if (osVersion != null) {
if (osVariant != null) {
userAgent += ` (Linux; ${osVersion}; ${osVariant})`;
} else {
userAgent += ` (Linux; ${osVersion})`;
}
}
export const request = Bluebird.promisifyAll(requestHandle, {
multiArgs: true,
}) as PromisifiedRequest;
export const resumable = resumableRequestLib.defaults(resumableOpts);
const requestOpts: requestLib.CoreOptions = {
gzip: true,
timeout: DEFAULT_REQUEST_TIMEOUT,
headers: {
'User-Agent': userAgent,
},
};
const resumableOpts = {
timeout: DEFAULT_REQUEST_TIMEOUT,
maxRetries: DEFAULT_REQUEST_RETRY_COUNT,
retryInterval: DEFAULT_REQUEST_RETRY_INTERVAL,
};
const requestHandle = requestLib.defaults(exports.requestOpts);
const request = Bluebird.promisifyAll(requestHandle, {
multiArgs: true,
}) as PromisifiedRequest;
const resumable = resumableRequestLib.defaults(resumableOpts);
return {
requestOpts,
request,
resumable,
};
});
export const getRequestInstance = once(async () => {
return (await getRequestInstances()).request;
});
export const getRequestOptions = once(async () => {
return (await getRequestInstances()).requestOpts;
});
export const getResumableRequest = once(async () => {
return (await getRequestInstances()).resumable;
});

View File

@ -595,10 +595,12 @@ module.exports = class Proxyvisor
"#{constants.proxyvisorHookReceiver}/v1/devices/"
sendUpdate: (device, timeout, endpoint) =>
request.putAsync "#{endpoint}#{device.uuid}", {
json: true
body: device.target
}
Promise.resolve(request.getRequestInstance())
.then (instance) ->
instance.putAsync "#{endpoint}#{device.uuid}", {
json: true
body: device.target
}
.timeout(timeout)
.spread (response, body) =>
if response.statusCode == 200
@ -610,7 +612,9 @@ module.exports = class Proxyvisor
return log.error("Error updating device #{device.uuid}", err)
sendDeleteHook: ({ uuid }, timeout, endpoint) =>
request.delAsync("#{endpoint}#{uuid}")
Promise.resolve(request.getRequestInstance())
.then (instance) ->
instance.delAsync("#{endpoint}#{uuid}")
.timeout(timeout)
.spread (response, body) =>
if response.statusCode == 200