Refine update locking interface

* Remove Supervisor lockfile cleanup SIGTERM listener
* Modify lockfile.getLocksTaken to read files from the filesystem
* Remove in-memory tracking of locks taken in favor of filesystem
* Require both `(resin-)updates.lock` to be locked with `nobody` UID
  for service to count as locked by the Supervisor

Signed-off-by: Christina Ying Wang <christina@balena.io>
This commit is contained in:
Christina Ying Wang 2024-03-13 00:10:08 -07:00
parent 10f294cf8e
commit fb1bd33ab6
13 changed files with 329 additions and 171 deletions

View File

@ -80,10 +80,6 @@ modprobe ip6_tables || true
export BASE_LOCK_DIR="/tmp/balena-supervisor/services" export BASE_LOCK_DIR="/tmp/balena-supervisor/services"
export LOCKFILE_UID=65534 export LOCKFILE_UID=65534
# Cleanup leftover Supervisor-created lockfiles from any previous processes.
# Supervisor-created lockfiles have a UID of 65534.
find "${ROOT_MOUNTPOINT}${BASE_LOCK_DIR}" -type f -user "${LOCKFILE_UID}" -name "*updates.lock" -delete || true
if [ "${LIVEPUSH}" = "1" ]; then if [ "${LIVEPUSH}" = "1" ]; then
exec npx nodemon --watch src --watch typings --ignore tests -e js,ts,json \ exec npx nodemon --watch src --watch typings --ignore tests -e js,ts,json \
--exec node -r ts-node/register/transpile-only src/app.ts --exec node -r ts-node/register/transpile-only src/app.ts

View File

@ -17,7 +17,7 @@ import {
ContractViolationError, ContractViolationError,
InternalInconsistencyError, InternalInconsistencyError,
} from '../lib/errors'; } from '../lib/errors';
import { getServicesLockedByAppId } from '../lib/update-lock'; import { getServicesLockedByAppId, LocksTakenMap } from '../lib/update-lock';
import { checkTruthy } from '../lib/validation'; import { checkTruthy } from '../lib/validation';
import App from './app'; import App from './app';
@ -150,7 +150,7 @@ export async function getRequiredSteps(
downloading, downloading,
availableImages, availableImages,
containerIdsByAppId, containerIdsByAppId,
locksTaken: getServicesLockedByAppId(), locksTaken: await getServicesLockedByAppId(),
}); });
} }
@ -168,7 +168,7 @@ export async function inferNextSteps(
containerIdsByAppId = {} as { containerIdsByAppId = {} as {
[appId: number]: UpdateState['containerIds']; [appId: number]: UpdateState['containerIds'];
}, },
locksTaken = getServicesLockedByAppId(), locksTaken = new LocksTakenMap(),
} = {}, } = {},
) { ) {
const currentAppIds = Object.keys(currentApps).map((i) => parseInt(i, 10)); const currentAppIds = Object.keys(currentApps).map((i) => parseInt(i, 10));

View File

@ -56,7 +56,7 @@ export async function get<T extends SchemaTypeKey>(
): Promise<SchemaReturn<T>> { ): Promise<SchemaReturn<T>> {
const $db = trx || db.models; const $db = trx || db.models;
if (Object.prototype.hasOwnProperty.call(Schema.schema, key)) { if (Object.hasOwn(Schema.schema, key)) {
const schemaKey = key as Schema.SchemaKey; const schemaKey = key as Schema.SchemaKey;
return getSchema(schemaKey, $db).then((value) => { return getSchema(schemaKey, $db).then((value) => {
@ -82,7 +82,7 @@ export async function get<T extends SchemaTypeKey>(
// the type system happy // the type system happy
return checkValueDecode(decoded, key, value) && decoded.right; return checkValueDecode(decoded, key, value) && decoded.right;
}); });
} else if (Object.prototype.hasOwnProperty.call(FnSchema.fnSchema, key)) { } else if (Object.hasOwn(FnSchema.fnSchema, key)) {
const fnKey = key as FnSchema.FnSchemaKey; const fnKey = key as FnSchema.FnSchemaKey;
// Cast the promise as something that produces an unknown, and this means that // Cast the promise as something that produces an unknown, and this means that
// we can validate the output of the function as well, ensuring that the type matches // we can validate the output of the function as well, ensuring that the type matches
@ -269,7 +269,7 @@ function validateConfigMap<T extends SchemaTypeKey>(
// throw if any value fails verification // throw if any value fails verification
return _.mapValues(configMap, (value, key) => { return _.mapValues(configMap, (value, key) => {
if ( if (
!Object.prototype.hasOwnProperty.call(Schema.schema, key) || !Object.hasOwn(Schema.schema, key) ||
!Schema.schema[key as Schema.SchemaKey].mutable !Schema.schema[key as Schema.SchemaKey].mutable
) { ) {
throw new Error( throw new Error(

View File

@ -12,8 +12,8 @@ import * as imageManager from '../compose/images';
import { import {
AppsJsonParseError, AppsJsonParseError,
EISDIR, isEISDIR,
ENOENT, isENOENT,
InternalInconsistencyError, InternalInconsistencyError,
} from '../lib/errors'; } from '../lib/errors';
import log from '../lib/supervisor-console'; import log from '../lib/supervisor-console';
@ -163,7 +163,7 @@ export async function loadTargetFromFile(appsPath: string): Promise<boolean> {
// It can be an empty path because if the file does not exist // It can be an empty path because if the file does not exist
// on host, the docker daemon creates an empty directory when // on host, the docker daemon creates an empty directory when
// the bind mount is added // the bind mount is added
if (ENOENT(e) || EISDIR(e)) { if (isENOENT(e) || isEISDIR(e)) {
log.debug('No apps.json file present, skipping preload'); log.debug('No apps.json file present, skipping preload');
} else { } else {
log.debug(e.message); log.debug(e.message);

View File

@ -6,7 +6,7 @@ import path from 'path';
import * as config from './config'; import * as config from './config';
import * as applicationManager from './compose/application-manager'; import * as applicationManager from './compose/application-manager';
import * as dbus from './lib/dbus'; import * as dbus from './lib/dbus';
import { ENOENT } from './lib/errors'; import { isENOENT } from './lib/errors';
import { mkdirp, unlinkAll } from './lib/fs-utils'; import { mkdirp, unlinkAll } from './lib/fs-utils';
import { import {
writeToBoot, writeToBoot,
@ -66,8 +66,8 @@ async function readProxy(): Promise<ProxyConfig | undefined> {
let redsocksConf: string; let redsocksConf: string;
try { try {
redsocksConf = await readFromBoot(redsocksConfPath, 'utf-8'); redsocksConf = await readFromBoot(redsocksConfPath, 'utf-8');
} catch (e: any) { } catch (e: unknown) {
if (!ENOENT(e)) { if (!isENOENT(e)) {
throw e; throw e;
} }
return; return;
@ -97,8 +97,8 @@ async function readProxy(): Promise<ProxyConfig | undefined> {
if (noProxy.length) { if (noProxy.length) {
conf.noProxy = noProxy; conf.noProxy = noProxy;
} }
} catch (e: any) { } catch (e: unknown) {
if (!ENOENT(e)) { if (!isENOENT(e)) {
throw e; throw e;
} }
} }

View File

@ -40,23 +40,27 @@ export class BadRequestError extends StatusError {
export const isBadRequestError = (e: unknown): e is BadRequestError => export const isBadRequestError = (e: unknown): e is BadRequestError =>
isStatusError(e) && e.statusCode === 400; isStatusError(e) && e.statusCode === 400;
export class DeviceNotFoundError extends TypedError {}
interface CodedSysError extends Error { interface CodedSysError extends Error {
code?: string; code?: string;
} }
export class DeviceNotFoundError extends TypedError {} const isCodedSysError = (e: unknown): e is CodedSysError =>
// See https://mdn.io/hasOwn
e != null && e instanceof Error && Object.hasOwn(e, 'code');
export function ENOENT(err: CodedSysError): boolean { export const isENOENT = (e: unknown): e is CodedSysError =>
return err.code === 'ENOENT'; isCodedSysError(e) && e.code === 'ENOENT';
}
export function EEXIST(err: CodedSysError): boolean { export const isEEXIST = (e: unknown): e is CodedSysError =>
return err.code === 'EEXIST'; isCodedSysError(e) && e.code === 'EEXIST';
}
export function EISDIR(err: CodedSysError): boolean { export const isEISDIR = (e: unknown): e is CodedSysError =>
return err.code === 'EISDIR'; isCodedSysError(e) && e.code === 'EISDIR';
}
export const isEPERM = (e: unknown): e is CodedSysError =>
isCodedSysError(e) && e.code === 'EPERM';
export function UnitNotLoadedError(err: string[]): boolean { export function UnitNotLoadedError(err: string[]): boolean {
return endsWith(err[0], 'not loaded.'); return endsWith(err[0], 'not loaded.');

View File

@ -3,6 +3,7 @@ import path from 'path';
import { exec as execSync } from 'child_process'; import { exec as execSync } from 'child_process';
import { promisify } from 'util'; import { promisify } from 'util';
import { uptime } from 'os'; import { uptime } from 'os';
import { isENOENT } from './errors';
export const exec = promisify(execSync); export const exec = promisify(execSync);
@ -76,7 +77,7 @@ export const touch = (file: string, time = new Date()) =>
fs.utimes(file, time, time).catch((e) => fs.utimes(file, time, time).catch((e) =>
// only create the file if it doesn't exist, // only create the file if it doesn't exist,
// if some other error happens is probably better to not touch it // if some other error happens is probably better to not touch it
e.code === 'ENOENT' isENOENT(e)
? fs ? fs
.open(file, 'w') .open(file, 'w')
.then((fd) => fd.close()) .then((fd) => fd.close())

View File

@ -1,34 +1,44 @@
import { promises as fs, unlinkSync, rmdirSync } from 'fs'; import { promises as fs } from 'fs';
import type { Stats, Dirent } from 'fs';
import os from 'os'; import os from 'os';
import { dirname } from 'path'; import { dirname } from 'path';
import { exec } from './fs-utils'; import { exec } from './fs-utils';
import { isENOENT, isEISDIR, isEPERM } from './errors';
// Equivalent to `drwxrwxrwt` // Equivalent to `drwxrwxrwt`
const STICKY_WRITE_PERMISSIONS = 0o1777; const STICKY_WRITE_PERMISSIONS = 0o1777;
/** // Returns all current locks taken under a directory (default: /tmp)
* Internal lockfile manager to track files in memory
*/
// Track locksTaken, so that the proper locks can be cleaned up on process exit
const locksTaken: { [lockName: string]: boolean } = {};
// Returns all current locks taken, as they've been stored in-memory.
// Optionally accepts filter function for only getting locks that match a condition. // Optionally accepts filter function for only getting locks that match a condition.
export const getLocksTaken = ( // A file is counted as a lock by default if it ends with `.lock`.
lockFilter: (path: string) => boolean = () => true, export const getLocksTaken = async (
): string[] => Object.keys(locksTaken).filter(lockFilter); rootDir: string = '/tmp',
lockFilter: (path: string, stat: Stats) => boolean = (p) =>
// Try to clean up any existing locks when the process exits p.endsWith('.lock'),
process.on('exit', () => { ): Promise<string[]> => {
for (const lockName of getLocksTaken()) { const locksTaken: string[] = [];
let filesOrDirs: Dirent[] = [];
try { try {
unlockSync(lockName); filesOrDirs = await fs.readdir(rootDir, { withFileTypes: true });
} catch (e) { } catch (err) {
// Ignore unlocking errors // If lockfile directory doesn't exist, no locks are taken
if (isENOENT(err)) {
return locksTaken;
} }
} }
}); for (const fileOrDir of filesOrDirs) {
const lockPath = `${rootDir}/${fileOrDir.name}`;
// A lock is taken if it's a file or directory within rootDir that passes filter fn
if (lockFilter(lockPath, await fs.stat(lockPath))) {
locksTaken.push(lockPath);
// Otherwise, if non-lock directory, seek locks recursively within directory
} else if (fileOrDir.isDirectory()) {
locksTaken.push(...(await getLocksTaken(lockPath, lockFilter)));
}
}
return locksTaken;
};
interface ChildProcessError { interface ChildProcessError {
code: number; code: number;
@ -77,8 +87,6 @@ export async function lock(path: string, uid: number = os.userInfo().uid) {
try { try {
// Lock the file using binary // Lock the file using binary
await exec(`lockfile -r 0 ${path}`, { uid }); await exec(`lockfile -r 0 ${path}`, { uid });
// Store a lock in memory as taken
locksTaken[path] = true;
} catch (error) { } catch (error) {
// Code 73 refers to EX_CANTCREAT (73) in sysexits.h, or: // Code 73 refers to EX_CANTCREAT (73) in sysexits.h, or:
// A (user specified) output file cannot be created. // A (user specified) output file cannot be created.
@ -110,7 +118,7 @@ export async function unlock(path: string): Promise<void> {
// Removing the lockfile releases the lock // Removing the lockfile releases the lock
await fs.unlink(path).catch((e) => { await fs.unlink(path).catch((e) => {
// if the error is EPERM|EISDIR, the file is a directory // if the error is EPERM|EISDIR, the file is a directory
if (e.code === 'EPERM' || e.code === 'EISDIR') { if (isEPERM(e) || isEISDIR(e)) {
return fs.rmdir(path).catch(() => { return fs.rmdir(path).catch(() => {
// if the directory is not empty or something else // if the directory is not empty or something else
// happens, ignore // happens, ignore
@ -119,17 +127,4 @@ export async function unlock(path: string): Promise<void> {
// If the file does not exist or some other error // If the file does not exist or some other error
// happens, then ignore the error // happens, then ignore the error
}); });
// Remove lockfile's in-memory tracking of a file
delete locksTaken[path];
}
export function unlockSync(path: string) {
try {
return unlinkSync(path);
} catch (e: any) {
if (e.code === 'EPERM' || e.code === 'EISDIR') {
return rmdirSync(path);
}
throw e;
}
} }

View File

@ -1,9 +1,10 @@
import { promises as fs } from 'fs'; import { promises as fs } from 'fs';
import path from 'path'; import path from 'path';
import type { Stats } from 'fs';
import { isRight } from 'fp-ts/lib/Either'; import { isRight } from 'fp-ts/lib/Either';
import { import {
ENOENT, isENOENT,
UpdatesLockedError, UpdatesLockedError,
InternalInconsistencyError, InternalInconsistencyError,
} from './errors'; } from './errors';
@ -65,10 +66,10 @@ async function dispose(
appIdentifier: string | number, appIdentifier: string | number,
release: () => void, release: () => void,
): Promise<void> { ): Promise<void> {
const locks = lockfile.getLocksTaken((p: string) =>
p.includes(`${BASE_LOCK_DIR}/${appIdentifier}`),
);
try { try {
const locks = await getLocksTaken(
pathOnRoot(`${BASE_LOCK_DIR}/${appIdentifier}`),
);
// Try to unlock all locks taken // Try to unlock all locks taken
await Promise.all(locks.map((l) => lockfile.unlock(l))); await Promise.all(locks.map((l) => lockfile.unlock(l)));
} finally { } finally {
@ -89,11 +90,12 @@ export async function takeLock(
const release = await takeGlobalLockRW(appId); const release = await takeGlobalLockRW(appId);
try { try {
const actuallyLocked: string[] = []; const actuallyLocked: string[] = [];
const locksTaken = await getServicesLockedByAppId();
// Filter out services that already have Supervisor-taken locks. // Filter out services that already have Supervisor-taken locks.
// This needs to be done after taking the appId write lock to avoid // This needs to be done after taking the appId write lock to avoid
// race conditions with locking. // race conditions with locking.
const servicesWithoutLock = services.filter( const servicesWithoutLock = services.filter(
(svc) => !getServicesLockedByAppId().isLocked(appId, svc), (svc) => !locksTaken.isLocked(appId, svc),
); );
for (const service of servicesWithoutLock) { for (const service of servicesWithoutLock) {
await mkdirp(pathOnRoot(lockPath(appId, service))); await mkdirp(pathOnRoot(lockPath(appId, service)));
@ -116,17 +118,18 @@ export async function releaseLock(appId: number) {
} }
/** /**
* Given a lockfile path `p`, return a tuple [appId, serviceName] of that path. * Given a lockfile path `p`, return an array [appId, serviceName, filename] of that path.
* Paths are assumed to end in the format /:appId/:serviceName/(resin-)updates.lock. * Paths are assumed to end in the format /:appId/:serviceName/(resin-)updates.lock.
*/ */
function getIdentifiersFromPath(p: string) { function getIdentifiersFromPath(p: string) {
const parts = p.split('/'); const parts = p.split('/');
if (parts.pop()?.match(/updates\.lock/) === null) { const filename = parts.pop();
if (filename?.match(/updates\.lock/) === null) {
return []; return [];
} }
const serviceName = parts.pop(); const serviceName = parts.pop();
const appId = parts.pop(); const appId = parts.pop();
return [appId, serviceName]; return [appId, serviceName, filename];
} }
type LockedEntity = { appId: number; services: string[] }; type LockedEntity = { appId: number; services: string[] };
@ -175,19 +178,62 @@ export class LocksTakenMap extends Map<number, Set<string>> {
} }
} }
// A wrapper function for lockfile.getLocksTaken that filters for Supervisor-taken locks.
// Exported for tests only; getServicesLockedByAppId is the intended public interface.
export async function getLocksTaken(
rootDir: string = pathOnRoot(BASE_LOCK_DIR),
): Promise<string[]> {
return await lockfile.getLocksTaken(
rootDir,
(p: string, s: Stats) =>
p.endsWith('updates.lock') && s.uid === LOCKFILE_UID,
);
}
/** /**
* Return a list of services that are locked by the Supervisor under each appId. * Return a list of services that are locked by the Supervisor under each appId.
* Both `resin-updates.lock` and `updates.lock` should be present per
* [appId, serviceName] pair for a service to be considered locked.
*/ */
export function getServicesLockedByAppId(): LocksTakenMap { export async function getServicesLockedByAppId(): Promise<LocksTakenMap> {
const locksTaken = lockfile.getLocksTaken(); const locksTaken = await getLocksTaken();
const servicesByAppId = new LocksTakenMap(); // Group locksTaken paths by appId & serviceName.
// filesTakenByAppId is of type Map<appId, Map<serviceName, Set<filename>>>
// and represents files taken under every [appId, serviceName] pair.
const filesTakenByAppId = new Map<number, Map<string, Set<string>>>();
for (const lockTakenPath of locksTaken) { for (const lockTakenPath of locksTaken) {
const [appId, serviceName] = getIdentifiersFromPath(lockTakenPath); const [appId, serviceName, filename] =
if (!StringIdentifier.is(appId) || !DockerName.is(serviceName)) { getIdentifiersFromPath(lockTakenPath);
if (
!StringIdentifier.is(appId) ||
!DockerName.is(serviceName) ||
!filename?.match(/updates\.lock/)
) {
continue; continue;
} }
const numAppId = +appId; const numAppId = +appId;
servicesByAppId.add(numAppId, serviceName); if (!filesTakenByAppId.has(numAppId)) {
filesTakenByAppId.set(numAppId, new Map());
}
const servicesTaken = filesTakenByAppId.get(numAppId)!;
if (!servicesTaken.has(serviceName)) {
servicesTaken.set(serviceName, new Set());
}
servicesTaken.get(serviceName)!.add(filename);
}
// Construct a LocksTakenMap from filesTakenByAppId, which represents
// services locked by the Supervisor.
const servicesByAppId = new LocksTakenMap();
for (const [appId, servicesTaken] of filesTakenByAppId) {
for (const [serviceName, filenames] of servicesTaken) {
if (
filenames.has('resin-updates.lock') &&
filenames.has('updates.lock')
) {
servicesByAppId.add(appId, serviceName);
}
}
} }
return servicesByAppId; return servicesByAppId;
} }
@ -228,7 +274,7 @@ export async function lock<T>(
releases.set(id, await takeGlobalLockRW(id)); releases.set(id, await takeGlobalLockRW(id));
// Get list of service folders in lock directory // Get list of service folders in lock directory
const serviceFolders = await fs.readdir(lockDir).catch((e) => { const serviceFolders = await fs.readdir(lockDir).catch((e) => {
if (ENOENT(e)) { if (isENOENT(e)) {
return []; return [];
} }
throw e; throw e;

View File

@ -5,7 +5,7 @@ import os from 'os';
import url from 'url'; import url from 'url';
import * as constants from './lib/constants'; import * as constants from './lib/constants';
import { EEXIST } from './lib/errors'; import { isEEXIST } from './lib/errors';
import { checkFalsey } from './lib/validation'; import { checkFalsey } from './lib/validation';
import blink = require('./lib/blink'); import blink = require('./lib/blink');
@ -71,7 +71,7 @@ export const startConnectivityCheck = _.once(
try { try {
await fs.mkdir(constants.vpnStatusPath); await fs.mkdir(constants.vpnStatusPath);
} catch (err: any) { } catch (err: any) {
if (EEXIST(err)) { if (isEEXIST(err)) {
log.debug('VPN status path exists.'); log.debug('VPN status path exists.');
} else { } else {
throw err; throw err;

View File

@ -1,5 +1,5 @@
import { expect } from 'chai'; import { expect } from 'chai';
import { promises as fs, mkdirSync } from 'fs'; import { promises as fs } from 'fs';
import type { TestFs } from 'mocha-pod'; import type { TestFs } from 'mocha-pod';
import { testfs } from 'mocha-pod'; import { testfs } from 'mocha-pod';
import * as os from 'os'; import * as os from 'os';
@ -148,54 +148,82 @@ describe('lib/lockfile', () => {
await expect(fs.access(lock)).to.be.rejected; await expect(fs.access(lock)).to.be.rejected;
}); });
it('should synchronously unlock a lockfile', () => { it('should get locks taken with default args', async () => {
const lock = path.join(lockdir, 'other.lock'); // Set up lock dirs
await fs.mkdir(`${lockdir}/1/main`, { recursive: true });
await fs.mkdir(`${lockdir}/2/aux`, { recursive: true });
lockfile.unlockSync(lock); // Take some locks
const locks = [
`${lockdir}/updates.lock`,
`${lockdir}/two.lock`,
`${lockdir}/1/main/updates.lock`,
`${lockdir}/1/main/resin-updates.lock`,
`${lockdir}/2/aux/updates.lock`,
`${lockdir}/2/aux/resin-updates.lock`,
];
await Promise.all(locks.map((lock) => lockfile.lock(lock)));
// Verify lockfile does not exist // Assert all locks are listed as taken
return expect(fs.access(lock)).to.be.rejected; expect(await lockfile.getLocksTaken(lockdir)).to.have.members(
locks.concat([`${lockdir}/other.lock`]),
);
// Clean up locks
await fs.rm(`${lockdir}`, { recursive: true });
}); });
it('should synchronously unlock a lockfile dir', () => { it('should get locks taken with a custom filter', async () => {
const lock = path.join(lockdir, 'update.lock'); // Set up lock dirs
await fs.mkdir(`${lockdir}/1`, { recursive: true });
await fs.mkdir(`${lockdir}/services/main`, { recursive: true });
await fs.mkdir(`${lockdir}/services/aux`, { recursive: true });
mkdirSync(lock, { recursive: true }); // Take some locks...
// - with a specific UID
lockfile.unlockSync(lock); await lockfile.lock(`${lockdir}/updates.lock`, NOBODY_UID);
// - as a directory
// Verify lockfile does not exist await fs.mkdir(`${lockdir}/1/updates.lock`);
return expect(fs.access(lock)).to.be.rejected; // - as a directory with a specific UID
}); await fs.mkdir(`${lockdir}/1/resin-updates.lock`);
await fs.chown(`${lockdir}/1/resin-updates.lock`, NOBODY_UID, NOBODY_UID);
it('should try to clean up existing locks on process exit', async () => { // - under a different root dir from default
// Create lockfiles await lockfile.lock(`${lockdir}/services/main/updates.lock`);
const lockOne = path.join(lockdir, 'updates.lock'); await lockfile.lock(`${lockdir}/services/aux/resin-updates.lock`);
const lockTwo = path.join(lockdir, 'two.lock');
await expect(lockfile.lock(lockOne)).to.not.be.rejected;
await expect(lockfile.lock(lockTwo, NOBODY_UID)).to.not.be.rejected;
// @ts-expect-error simulate process exit event
process.emit('exit');
// Verify lockfile removal regardless of appId / appUuid
await expect(fs.access(lockOne)).to.be.rejected;
await expect(fs.access(lockTwo)).to.be.rejected;
});
it('allows to list locks taken according to a filter function', async () => {
// Create multiple lockfiles
const lockOne = path.join(lockdir, 'updates.lock');
const lockTwo = path.join(lockdir, 'two.lock');
await expect(lockfile.lock(lockOne)).to.not.be.rejected;
await expect(lockfile.lock(lockTwo, NOBODY_UID)).to.not.be.rejected;
// Assert appropriate locks are listed as taken...
// - with a specific UID
expect( expect(
lockfile.getLocksTaken((filepath) => filepath.includes('lockdir')), await lockfile.getLocksTaken(
).to.have.members([lockOne, lockTwo]); lockdir,
(p, stats) => p.endsWith('.lock') && stats.uid === NOBODY_UID,
),
).to.have.members([
`${lockdir}/updates.lock`,
`${lockdir}/1/resin-updates.lock`,
`${lockdir}/other.lock`,
]);
// - as a directory
expect( expect(
lockfile.getLocksTaken((filepath) => filepath.includes('two')), await lockfile.getLocksTaken(
).to.have.members([lockTwo]); lockdir,
expect(lockfile.getLocksTaken()).to.have.members([lockOne, lockTwo]); (p, stats) => p.endsWith('.lock') && stats.isDirectory(),
),
).to.have.members([
`${lockdir}/1/updates.lock`,
`${lockdir}/1/resin-updates.lock`,
]);
// - under a different root dir from default
expect(
await lockfile.getLocksTaken(`${lockdir}/services`, (p) =>
p.endsWith('.lock'),
),
).to.have.members([
`${lockdir}/services/main/updates.lock`,
`${lockdir}/services/aux/resin-updates.lock`,
]);
// Clean up locks
await fs.rm(`${lockdir}`, { recursive: true });
}); });
}); });

View File

@ -70,13 +70,16 @@ describe('lib/update-lock', () => {
const takeLocks = () => const takeLocks = () =>
Promise.all( Promise.all(
supportedLockfiles.map((lf) => supportedLockfiles.map((lf) =>
lockfile.lock(path.join(lockdir(testAppId, testServiceName), lf)), lockfile.lock(
path.join(lockdir(testAppId, testServiceName), lf),
updateLock.LOCKFILE_UID,
),
), ),
); );
const releaseLocks = async () => { const releaseLocks = async () => {
await Promise.all( await Promise.all(
lockfile.getLocksTaken().map((lock) => lockfile.unlock(lock)), (await updateLock.getLocksTaken()).map((lock) => lockfile.unlock(lock)),
); );
// Remove any other lockfiles created for the testAppId // Remove any other lockfiles created for the testAppId
@ -150,8 +153,8 @@ describe('lib/update-lock', () => {
) )
.catch((err) => expect(err).to.be.instanceOf(UpdatesLockedError)); .catch((err) => expect(err).to.be.instanceOf(UpdatesLockedError));
// Since the lock-taking failed, there should be no locks to dispose of // Since the lock-taking with `nobody` uid failed, there should be no locks to dispose of
expect(lockfile.getLocksTaken()).to.have.length(0); expect(await updateLock.getLocksTaken()).to.have.length(0);
// Restore the locks that were taken at the beginning of the test // Restore the locks that were taken at the beginning of the test
await releaseLocks(); await releaseLocks();
@ -286,23 +289,96 @@ describe('lib/update-lock', () => {
}); });
}); });
describe('getLocksTaken', () => {
const lockdir = pathOnRoot(updateLock.BASE_LOCK_DIR);
before(async () => {
await testfs({
[lockdir]: {},
}).enable();
// TODO: enable mocha-pod to work with empty directories
await fs.mkdir(`${lockdir}/123/main`, { recursive: true });
await fs.mkdir(`${lockdir}/123/aux`, { recursive: true });
await fs.mkdir(`${lockdir}/123/invalid`, { recursive: true });
});
after(async () => {
await fs.rm(`${lockdir}/123`, { recursive: true });
await testfs.restore();
});
it('resolves with all locks taken with the Supervisor lockfile UID', async () => {
// Set up valid lockfiles including some directories
await Promise.all(
['resin-updates.lock', 'updates.lock'].map((lf) => {
const p = `${lockdir}/123/main/${lf}`;
return fs
.mkdir(p)
.then(() =>
fs.chown(p, updateLock.LOCKFILE_UID, updateLock.LOCKFILE_UID),
);
}),
);
await Promise.all([
lockfile.lock(
`${lockdir}/123/aux/updates.lock`,
updateLock.LOCKFILE_UID,
),
lockfile.lock(
`${lockdir}/123/aux/resin-updates.lock`,
updateLock.LOCKFILE_UID,
),
]);
// Set up invalid lockfiles with root UID
await Promise.all(
['resin-updates.lock', 'updates.lock'].map((lf) =>
lockfile.lock(`${lockdir}/123/invalid/${lf}`),
),
);
const locksTaken = await updateLock.getLocksTaken();
expect(locksTaken).to.have.length(4);
expect(locksTaken).to.deep.include.members([
`${lockdir}/123/aux/resin-updates.lock`,
`${lockdir}/123/aux/updates.lock`,
`${lockdir}/123/main/resin-updates.lock`,
`${lockdir}/123/main/updates.lock`,
]);
expect(locksTaken).to.not.deep.include.members([
`${lockdir}/123/invalid/resin-updates.lock`,
`${lockdir}/123/invalid/updates.lock`,
]);
});
});
describe('getServicesLockedByAppId', () => { describe('getServicesLockedByAppId', () => {
const validPaths = [ const lockdir = pathOnRoot(updateLock.BASE_LOCK_DIR);
'/tmp/123/one/updates.lock', const validDirs = [
'/tmp/123/two/updates.lock', `${lockdir}/123/one`,
'/tmp/123/three/updates.lock', `${lockdir}/123/two`,
'/tmp/balena-supervisor/services/456/server/updates.lock', `${lockdir}/123/three`,
'/tmp/balena-supervisor/services/456/client/updates.lock', `${lockdir}/456/server`,
'/tmp/balena-supervisor/services/789/main/resin-updates.lock', `${lockdir}/456/client`,
`${lockdir}/789/main`,
]; ];
const validPaths = ['resin-updates.lock', 'updates.lock']
.map((lf) => validDirs.map((d) => path.join(d, lf)))
.flat();
const invalidPaths = [ const invalidPaths = [
'/tmp/balena-supervisor/services/456/updates.lock', // No appId
'/tmp/balena-supervisor/services/server/updates.lock', `${lockdir}/456/updates.lock`,
'/tmp/test/updates.lock', // No service
`${lockdir}/server/updates.lock`,
// No appId or service
`${lockdir}/test/updates.lock`,
// One of (resin-)updates.lock is missing
`${lockdir}/123/one/resin-updates.lock`,
`${lockdir}/123/two/updates.lock`,
]; ];
let tFs: TestFs.Enabled; let tFs: TestFs.Enabled;
beforeEach(async () => { beforeEach(async () => {
tFs = await testfs({ '/tmp': {} }).enable(); tFs = await testfs({
[lockdir]: {},
}).enable();
// TODO: mocha-pod should support empty directories // TODO: mocha-pod should support empty directories
await Promise.all( await Promise.all(
validPaths validPaths
@ -325,7 +401,7 @@ describe('lib/update-lock', () => {
validPaths.map((p) => lockfile.lock(p, updateLock.LOCKFILE_UID)), validPaths.map((p) => lockfile.lock(p, updateLock.LOCKFILE_UID)),
); );
const locksTakenMap = updateLock.getServicesLockedByAppId(); const locksTakenMap = await updateLock.getServicesLockedByAppId();
expect([...locksTakenMap.keys()]).to.deep.include.members([ expect([...locksTakenMap.keys()]).to.deep.include.members([
123, 456, 789, 123, 456, 789,
]); ]);
@ -348,9 +424,17 @@ describe('lib/update-lock', () => {
it('should ignore invalid lockfile locations', async () => { it('should ignore invalid lockfile locations', async () => {
// Set up lockfiles // Set up lockfiles
await Promise.all(invalidPaths.map((p) => lockfile.lock(p))); await Promise.all(
invalidPaths.map((p) => lockfile.lock(p, updateLock.LOCKFILE_UID)),
expect(updateLock.getServicesLockedByAppId().size).to.equal(0); );
// Take another lock with an invalid UID but with everything else
// (appId, service, both lockfiles present) correct
await Promise.all(
['resin-updates.lock', 'updates.lock'].map((lf) =>
lockfile.lock(path.join(`${lockdir}/789/main`, lf)),
),
);
expect((await updateLock.getServicesLockedByAppId()).size).to.equal(0);
// Cleanup lockfiles // Cleanup lockfiles
await Promise.all(invalidPaths.map((p) => lockfile.unlock(p))); await Promise.all(invalidPaths.map((p) => lockfile.unlock(p)));
@ -396,10 +480,10 @@ describe('lib/update-lock', () => {
// Take locks for appId 1 // Take locks for appId 1
await updateLock.takeLock(1, ['server', 'client']); await updateLock.takeLock(1, ['server', 'client']);
// Locks should have been taken // Locks should have been taken
expect(lockfile.getLocksTaken()).to.deep.include.members( expect(await updateLock.getLocksTaken()).to.deep.include.members(
serviceLockPaths[1], serviceLockPaths[1],
); );
expect(lockfile.getLocksTaken()).to.have.length(4); expect(await updateLock.getLocksTaken()).to.have.length(4);
expect( expect(
await fs.readdir(path.join(lockdir, '1', 'server')), await fs.readdir(path.join(lockdir, '1', 'server')),
).to.include.members(['updates.lock', 'resin-updates.lock']); ).to.include.members(['updates.lock', 'resin-updates.lock']);
@ -409,11 +493,11 @@ describe('lib/update-lock', () => {
// Take locks for appId 2 // Take locks for appId 2
await updateLock.takeLock(2, ['main']); await updateLock.takeLock(2, ['main']);
// Locks should have been taken for appid 1 & 2 // Locks should have been taken for appid 1 & 2
expect(lockfile.getLocksTaken()).to.deep.include.members([ expect(await updateLock.getLocksTaken()).to.deep.include.members([
...serviceLockPaths[1], ...serviceLockPaths[1],
...serviceLockPaths[2], ...serviceLockPaths[2],
]); ]);
expect(lockfile.getLocksTaken()).to.have.length(6); expect(await updateLock.getLocksTaken()).to.have.length(6);
expect( expect(
await fs.readdir(path.join(lockdir, '2', 'main')), await fs.readdir(path.join(lockdir, '2', 'main')),
).to.have.length(2); ).to.have.length(2);
@ -429,7 +513,7 @@ describe('lib/update-lock', () => {
// Take locks for app with nonexistent service directories // Take locks for app with nonexistent service directories
await updateLock.takeLock(3, ['api']); await updateLock.takeLock(3, ['api']);
// Locks should have been taken // Locks should have been taken
expect(lockfile.getLocksTaken()).to.deep.include( expect(await updateLock.getLocksTaken()).to.deep.include(
path.join(lockdir, '3', 'api', 'updates.lock'), path.join(lockdir, '3', 'api', 'updates.lock'),
path.join(lockdir, '3', 'api', 'resin-updates.lock'), path.join(lockdir, '3', 'api', 'resin-updates.lock'),
); );
@ -450,21 +534,21 @@ describe('lib/update-lock', () => {
it('should not take lock for services where Supervisor-taken lock already exists', async () => { it('should not take lock for services where Supervisor-taken lock already exists', async () => {
// Take locks for one service of appId 1 // Take locks for one service of appId 1
await lockfile.lock(serviceLockPaths[1][0]); await lockfile.lock(serviceLockPaths[1][0], updateLock.LOCKFILE_UID);
await lockfile.lock(serviceLockPaths[1][1]); await lockfile.lock(serviceLockPaths[1][1], updateLock.LOCKFILE_UID);
// Sanity check that locks are taken & tracked by Supervisor // Sanity check that locks are taken & tracked by Supervisor
expect(lockfile.getLocksTaken()).to.deep.include( expect(await updateLock.getLocksTaken()).to.deep.include(
serviceLockPaths[1][0], serviceLockPaths[1][0],
serviceLockPaths[1][1], serviceLockPaths[1][1],
); );
expect(lockfile.getLocksTaken()).to.have.length(2); expect(await updateLock.getLocksTaken()).to.have.length(2);
// Take locks using takeLock, should only lock service which doesn't // Take locks using takeLock, should only lock service which doesn't
// already have locks // already have locks
await expect( await expect(
updateLock.takeLock(1, ['server', 'client']), updateLock.takeLock(1, ['server', 'client']),
).to.eventually.deep.include.members(['client']); ).to.eventually.deep.include.members(['client']);
// Check that locks are taken // Check that locks are taken
expect(lockfile.getLocksTaken()).to.deep.include.members( expect(await updateLock.getLocksTaken()).to.deep.include.members(
serviceLockPaths[1], serviceLockPaths[1],
); );
// Clean up lockfiles // Clean up lockfiles
@ -483,7 +567,7 @@ describe('lib/update-lock', () => {
updateLock.takeLock(1, ['server', 'client']), updateLock.takeLock(1, ['server', 'client']),
).to.eventually.be.rejectedWith(UpdatesLockedError); ).to.eventually.be.rejectedWith(UpdatesLockedError);
// No Supervisor locks should have been taken // No Supervisor locks should have been taken
expect(lockfile.getLocksTaken()).to.have.length(0); expect(await updateLock.getLocksTaken()).to.have.length(0);
// Clean up user-created lockfiles // Clean up user-created lockfiles
for (const lockPath of serviceLockPaths[1]) { for (const lockPath of serviceLockPaths[1]) {
await fs.rm(lockPath); await fs.rm(lockPath);
@ -493,10 +577,10 @@ describe('lib/update-lock', () => {
updateLock.takeLock(1, ['server', 'client']), updateLock.takeLock(1, ['server', 'client']),
).to.eventually.not.be.rejectedWith(UpdatesLockedError); ).to.eventually.not.be.rejectedWith(UpdatesLockedError);
// Check that locks are taken // Check that locks are taken
expect(lockfile.getLocksTaken()).to.deep.include.members( expect(await updateLock.getLocksTaken()).to.deep.include.members(
serviceLockPaths[1], serviceLockPaths[1],
); );
expect(lockfile.getLocksTaken()).to.have.length(4); expect(await updateLock.getLocksTaken()).to.have.length(4);
// Clean up lockfiles // Clean up lockfiles
for (const lockPath of serviceLockPaths[1]) { for (const lockPath of serviceLockPaths[1]) {
await lockfile.unlock(lockPath); await lockfile.unlock(lockPath);
@ -510,13 +594,13 @@ describe('lib/update-lock', () => {
const takeLockPromise = updateLock.takeLock(1, ['server', 'client']); const takeLockPromise = updateLock.takeLock(1, ['server', 'client']);
// Locks should have not been taken even after waiting // Locks should have not been taken even after waiting
await setTimeout(500); await setTimeout(500);
expect(lockfile.getLocksTaken()).to.have.length(0); expect(await updateLock.getLocksTaken()).to.have.length(0);
// Release the write lock // Release the write lock
release(); release();
// Locks should be taken // Locks should be taken
await takeLockPromise; await takeLockPromise;
// Locks should have been taken // Locks should have been taken
expect(lockfile.getLocksTaken()).to.deep.include.members( expect(await updateLock.getLocksTaken()).to.deep.include.members(
serviceLockPaths[1], serviceLockPaths[1],
); );
}); });
@ -545,38 +629,42 @@ describe('lib/update-lock', () => {
it('releases locks for an appId', async () => { it('releases locks for an appId', async () => {
// Lock services for appId 1 // Lock services for appId 1
for (const lockPath of serviceLockPaths[1]) { for (const lockPath of serviceLockPaths[1]) {
await lockfile.lock(lockPath); await lockfile.lock(lockPath, updateLock.LOCKFILE_UID);
} }
// Sanity check that locks are taken & tracked by Supervisor // Sanity check that locks are taken & tracked by Supervisor
expect(lockfile.getLocksTaken()).to.deep.include.members( expect(await updateLock.getLocksTaken()).to.deep.include.members(
serviceLockPaths[1], serviceLockPaths[1],
); );
// Release locks for appId 1 // Release locks for appId 1
await updateLock.releaseLock(1); await updateLock.releaseLock(1);
// Locks should have been released // Locks should have been released
expect(lockfile.getLocksTaken()).to.have.length(0); expect(await updateLock.getLocksTaken()).to.have.length(0);
// Double check that the lockfiles are removed // Double check that the lockfiles are removed
expect(await fs.readdir(`${lockdir}/1/server`)).to.have.length(0); expect(await fs.readdir(`${lockdir}/1/server`)).to.have.length(0);
expect(await fs.readdir(`${lockdir}/1/client`)).to.have.length(0); expect(await fs.readdir(`${lockdir}/1/client`)).to.have.length(0);
}); });
it('does not error if there are no locks to release', async () => { it('does not error if there are no locks to release', async () => {
expect(lockfile.getLocksTaken()).to.have.length(0); expect(await updateLock.getLocksTaken()).to.have.length(0);
// Should not error // Should not error
await updateLock.releaseLock(1); await updateLock.releaseLock(1);
expect(lockfile.getLocksTaken()).to.have.length(0); expect(await updateLock.getLocksTaken()).to.have.length(0);
}); });
it('ignores locks outside of appId scope', async () => { it('ignores locks outside of appId scope', async () => {
const lockPath = `${lockdir}/2/main/updates.lock`; const lockPath = `${lockdir}/2/main/updates.lock`;
// Lock services outside of appId scope // Lock services outside of appId scope
await lockfile.lock(lockPath); await lockfile.lock(lockPath, updateLock.LOCKFILE_UID);
// Sanity check that locks are taken & tracked by Supervisor // Sanity check that locks are taken & tracked by Supervisor
expect(lockfile.getLocksTaken()).to.deep.include.members([lockPath]); expect(await updateLock.getLocksTaken()).to.deep.include.members([
lockPath,
]);
// Release locks for appId 1 // Release locks for appId 1
await updateLock.releaseLock(1); await updateLock.releaseLock(1);
// Locks for appId 2 should not have been released // Locks for appId 2 should not have been released
expect(lockfile.getLocksTaken()).to.deep.include.members([lockPath]); expect(await updateLock.getLocksTaken()).to.deep.include.members([
lockPath,
]);
// Double check that the lockfile is still there // Double check that the lockfile is still there
expect(await fs.readdir(`${lockdir}/2/main`)).to.have.length(1); expect(await fs.readdir(`${lockdir}/2/main`)).to.have.length(1);
// Clean up the lockfile // Clean up the lockfile
@ -586,10 +674,10 @@ describe('lib/update-lock', () => {
it('waits to release locks until resource write lock is taken', async () => { it('waits to release locks until resource write lock is taken', async () => {
// Lock services for appId 1 // Lock services for appId 1
for (const lockPath of serviceLockPaths[1]) { for (const lockPath of serviceLockPaths[1]) {
await lockfile.lock(lockPath); await lockfile.lock(lockPath, updateLock.LOCKFILE_UID);
} }
// Sanity check that locks are taken & tracked by Supervisor // Sanity check that locks are taken & tracked by Supervisor
expect(lockfile.getLocksTaken()).to.deep.include.members( expect(await updateLock.getLocksTaken()).to.deep.include.members(
serviceLockPaths[1], serviceLockPaths[1],
); );
// Take the write lock for appId 1 // Take the write lock for appId 1
@ -598,7 +686,7 @@ describe('lib/update-lock', () => {
const releaseLockPromise = updateLock.releaseLock(1); const releaseLockPromise = updateLock.releaseLock(1);
// Locks should have not been released even after waiting // Locks should have not been released even after waiting
await setTimeout(500); await setTimeout(500);
expect(lockfile.getLocksTaken()).to.deep.include.members( expect(await updateLock.getLocksTaken()).to.deep.include.members(
serviceLockPaths[1], serviceLockPaths[1],
); );
// Release the write lock // Release the write lock
@ -606,7 +694,7 @@ describe('lib/update-lock', () => {
// Release locks for appId 1 should resolve // Release locks for appId 1 should resolve
await releaseLockPromise; await releaseLockPromise;
// Locks should have been released // Locks should have been released
expect(lockfile.getLocksTaken()).to.have.length(0); expect(await updateLock.getLocksTaken()).to.have.length(0);
}); });
}); });
}); });

View File

@ -74,7 +74,7 @@ export function registerOverride<
} }
export function restoreOverride<T extends DockerodeFunction>(name: T) { export function restoreOverride<T extends DockerodeFunction>(name: T) {
if (Object.prototype.hasOwnProperty.call(overrides, name)) { if (Object.hasOwn(overrides, name)) {
delete overrides[name]; delete overrides[name];
} }
} }