diff --git a/src/application-manager.coffee b/src/application-manager.coffee index 1d25934f..04b18581 100644 --- a/src/application-manager.coffee +++ b/src/application-manager.coffee @@ -20,7 +20,7 @@ ServiceManager = require './compose/service-manager' { Images } = require './compose/images' { NetworkManager } = require './compose/network-manager' { Network } = require './compose/network' -Volumes = require './compose/volumes' +{ Volumes } = require './compose/volumes' Proxyvisor = require './proxyvisor' diff --git a/src/compose/volumes.coffee b/src/compose/volumes.coffee deleted file mode 100644 index 578a85e2..00000000 --- a/src/compose/volumes.coffee +++ /dev/null @@ -1,106 +0,0 @@ -Promise = require 'bluebird' -_ = require 'lodash' -path = require 'path' - -logTypes = require '../lib/log-types' -constants = require '../lib/constants' -{ checkInt } = require '../lib/validation' -{ NotFoundError } = require '../lib/errors' -{ defaultLegacyVolume } = require '../lib/migration' -{ safeRename } = require '../lib/fs-utils' -ComposeUtils = require './utils' - -module.exports = class Volumes - constructor: ({ @docker, @logger }) -> - - format: (volume) -> - m = volume.Name.match(/^([0-9]+)_(.+)$/) - appId = checkInt(m[1]) - name = m[2] - return { - name: name - appId: appId - config: { - labels: _.omit(ComposeUtils.normalizeLabels(volume.Labels), _.keys(constants.defaultVolumeLabels)) - driverOpts: volume.Options - } - handle: volume - } - - _listWithBothLabels: => - Promise.join( - @docker.listVolumes(filters: label: [ 'io.resin.supervised' ]) - @docker.listVolumes(filters: label: [ 'io.balena.supervised' ]) - (legacyVolumesResponse, currentVolumesResponse) -> - legacyVolumes = legacyVolumesResponse.Volumes ? [] - currentVolumes = currentVolumesResponse.Volumes ? [] - return _.unionBy(legacyVolumes, currentVolumes, 'Name') - ) - - getAll: => - @_listWithBothLabels() - .map (volume) => - @docker.getVolume(volume.Name).inspect() - .then(@format) - - getAllByAppId: (appId) => - @getAll() - .then (volumes) -> - _.filter(volumes, { appId }) - - get: ({ name, appId }) -> - @docker.getVolume("#{appId}_#{name}").inspect() - .then(@format) - - # TODO: what config values are relevant/whitelisted? - # For now we only care about driverOpts and labels - create: ({ name, config = {}, appId }) => - config = _.mapKeys(config, (v, k) -> _.camelCase(k)) - @logger.logSystemEvent(logTypes.createVolume, { volume: { name } }) - labels = _.clone(config.labels) ? {} - _.assign(labels, constants.defaultVolumeLabels) - driverOpts = config.driverOpts ? {} - - @get({ name, appId }) - .tap (vol) => - if !@isEqualConfig(vol.config, config) - throw new Error("Trying to create volume '#{name}', but a volume with same name and different configuration exists") - .catch NotFoundError, => - @docker.createVolume({ - Name: "#{appId}_#{name}" - Labels: labels - DriverOpts: driverOpts - }).call('inspect').then(@format) - .tapCatch (err) => - @logger.logSystemEvent(logTypes.createVolumeError, { volume: { name }, error: err }) - - createFromLegacy: (appId) => - name = defaultLegacyVolume() - legacyPath = path.join(constants.rootMountPoint, 'mnt/data/resin-data', appId.toString()) - @createFromPath({ name, appId }, legacyPath) - .catch (err) => - @logger.logSystemMessage("Warning: could not migrate legacy /data volume: #{err.message}", { error: err }, 'Volume migration error') - - # oldPath must be a path inside /mnt/data - createFromPath: ({ name, config = {}, appId }, oldPath) => - @create({ name, config, appId }) - .get('handle') - .then (v) -> - # Convert the path to be of the same mountpoint so that rename can work - volumePath = path.join(constants.rootMountPoint, 'mnt/data', v.Mountpoint.split(path.sep).slice(3)...) - safeRename(oldPath, volumePath) - - remove: ({ name, appId }) -> - @logger.logSystemEvent(logTypes.removeVolume, { volume: { name } }) - @docker.getVolume("#{appId}_#{name}").remove() - .catch (err) => - @logger.logSystemEvent(logTypes.removeVolumeError, { volume: { name, appId }, error: err }) - - isEqualConfig: (current = {}, target = {}) -> - current = _.mapKeys(current, (v, k) -> _.camelCase(k)) - target = _.mapKeys(target, (v, k) -> _.camelCase(k)) - currentOpts = current.driverOpts ? {} - targetOpts = target.driverOpts ? {} - currentLabels = current.labels ? {} - targetLabels = target.labels ? {} - return _.isEqual(currentLabels, targetLabels) and _.isEqual(currentOpts, targetOpts) diff --git a/src/compose/volumes.ts b/src/compose/volumes.ts new file mode 100644 index 00000000..64d79369 --- /dev/null +++ b/src/compose/volumes.ts @@ -0,0 +1,231 @@ +import * as Dockerode from 'dockerode'; +import * as _ from 'lodash'; +import * as path from 'path'; + +import Docker = require('../lib/docker-utils'); +import Logger from '../logger'; + +import constants = require('../lib/constants'); +import { InternalInconsistencyError, NotFoundError } from '../lib/errors'; +import { safeRename } from '../lib/fs-utils'; +import * as LogTypes from '../lib/log-types'; +import { defaultLegacyVolume } from '../lib/migration'; +import { LabelObject } from '../lib/types'; +import { checkInt } from '../lib/validation'; +import * as ComposeUtils from './utils'; + +interface VolumeConstructOpts { + docker: Docker; + logger: Logger; +} + +export interface ComposeVolume { + name: string; + appId: number; + config: { + labels: LabelObject; + driverOpts: Dockerode.VolumeInspectInfo['Options']; + }; + dockerVolume: Dockerode.VolumeInspectInfo; +} + +interface VolumeNameOpts { + name: string; + appId: number; +} + +// This weird type is currently needed because the create function (and helpers) +// accept either a docker volume or a compose volume (or an empty object too apparently). +// If we instead split the tasks into createFromCompose and createFromDocker, we will no +// longer have this issue (and weird typing) +type VolumeConfig = ComposeVolume['config'] | Dockerode.VolumeInspectInfo | {}; +type VolumeCreateOpts = VolumeNameOpts & { + config?: VolumeConfig; +}; + +export class Volumes { + private docker: Docker; + private logger: Logger; + + public constructor(opts: VolumeConstructOpts) { + this.docker = opts.docker; + this.logger = opts.logger; + } + + public async getAll(): Promise { + const volumes = await this.listWithBothLabels(); + return volumes.map(Volumes.format); + } + + public async getAllByAppId(appId: number): Promise { + const all = await this.getAll(); + return _.filter(all, { appId }); + } + + public async get({ name, appId }: VolumeNameOpts): Promise { + const volume = await this.docker.getVolume(`${appId}_${name}`).inspect(); + return Volumes.format(volume); + } + + public async create(opts: VolumeCreateOpts): Promise { + const { name, config = {}, appId } = opts; + const camelCaseConfig: Dictionary = _.mapKeys(config, (_v, k) => + _.camelCase(k), + ); + + this.logger.logSystemEvent(LogTypes.createVolume, { volume: { name } }); + + const labels = _.clone(camelCaseConfig.labels as LabelObject) || {}; + _.assign(labels, constants.defaultVolumeLabels); + + const driverOpts: Dictionary = + camelCaseConfig.driverOpts != null + ? (camelCaseConfig.driverOpts as Dictionary) + : {}; + + try { + const volume = await this.get({ name, appId }); + if (!this.isEqualConfig(volume.config, config)) { + throw new InternalInconsistencyError( + `Trying to create volume '${name}', but a volume with the same name and different configuration exists`, + ); + } + return volume; + } catch (e) { + if (!NotFoundError(e)) { + this.logger.logSystemEvent(LogTypes.createVolumeError, { + volume: { name }, + error: e, + }); + throw e; + } + const volume = await this.docker.createVolume({ + Name: Volumes.generateVolumeName({ name, appId }), + Labels: labels, + DriverOpts: driverOpts, + }); + + return Volumes.format(await volume.inspect()); + } + } + + public async createFromLegacy(appId: number): Promise { + const name = defaultLegacyVolume(); + const legacyPath = path.join( + constants.rootMountPoint, + 'mnt/data/resin-data', + appId.toString(), + ); + + try { + return await this.createFromPath({ name, appId }, legacyPath); + } catch (e) { + this.logger.logSystemMessage( + `Warning: could not migrate legacy /data volume: ${e.message}`, + { error: e }, + 'Volume migration error', + ); + } + } + + // oldPath must be a path inside /mnt/data + public async createFromPath( + opts: VolumeCreateOpts, + oldPath: string, + ): Promise { + const volume = await this.create(opts); + const handle = volume.dockerVolume; + + // Convert the path to be of the same mountpoint so that rename can work + const volumePath = path.join( + constants.rootMountPoint, + 'mnt/data', + ...handle.Mountpoint.split(path.sep).slice(3), + ); + await safeRename(oldPath, volumePath); + } + + public async remove({ name, appId }: VolumeNameOpts) { + this.logger.logSystemEvent(LogTypes.removeVolume, { volume: { name } }); + try { + await this.docker + .getVolume(Volumes.generateVolumeName({ name, appId })) + .remove(); + } catch (e) { + this.logger.logSystemEvent(LogTypes.removeVolumeError, { + volume: { name, appId }, + error: e, + }); + } + } + + public isEqualConfig(current: VolumeConfig, target: VolumeConfig): boolean { + const currentConfig = (_.mapKeys(current, (_v, k) => + _.camelCase(k), + ) as unknown) as ComposeVolume['config']; + const targetConfig = (_.mapKeys(target, (_v, k) => + _.camelCase(k), + ) as unknown) as ComposeVolume['config']; + + const currentOpts = currentConfig.driverOpts || {}; + const targetOpts = targetConfig.driverOpts || {}; + + const currentLabels = currentConfig.labels || {}; + const targetLabels = targetConfig.labels || {}; + + return ( + _.isEqual(currentOpts, targetOpts) && + _.isEqual(currentLabels, targetLabels) + ); + } + + private static format(volume: Dockerode.VolumeInspectInfo): ComposeVolume { + const match = volume.Name.match(/^([0-9]+)_(.+)$/); + if (match == null) { + throw new Error('Malformed volume name in Volume.format'); + } + const appId = checkInt(match[1]); + const name = match[2]; + + return { + name, + // We know this cast is fine due to the regex + appId: appId as number, + config: { + labels: _.omit( + ComposeUtils.normalizeLabels(volume.Labels), + _.keys(constants.defaultVolumeLabels), + ), + driverOpts: volume.Options, + }, + dockerVolume: volume, + }; + } + + private async listWithBothLabels(): Promise { + // We have to cast the listVolumes call from any[] to any below, until the + // relevant PR: https://github.com/DefinitelyTyped/DefinitelyTyped/pull/32383 + // is merged and released + const [legacyResponse, currentResponse]: [ + Dockerode.VolumeInfoList, + Dockerode.VolumeInfoList + ] = await Promise.all([ + this.docker.listVolumes({ + filters: { label: ['io.resin.supervised'] }, + }) as Promise, + this.docker.listVolumes({ + filters: { label: ['io.balena.supervised'] }, + }) as Promise, + ]); + + const legacyVolumes = _.get(legacyResponse, 'Volumes', []); + const currentVolumes = _.get(currentResponse, 'Volumes', []); + return _.unionBy(legacyVolumes, currentVolumes, 'Name'); + } + + private static generateVolumeName({ name, appId }: VolumeNameOpts) { + return `${appId}_${name}`; + } +} + +export default Volumes; diff --git a/typings/dockerode-ext.d.ts b/typings/dockerode-ext.d.ts index 2655c916..cc3c5ea9 100644 --- a/typings/dockerode-ext.d.ts +++ b/typings/dockerode-ext.d.ts @@ -21,4 +21,27 @@ declare module 'dockerode' { Healthcheck?: DockerHealthcheck; StopTimeout?: number; } + + // TODO: Once https://github.com/DefinitelyTyped/DefinitelyTyped/pull/32383 + // is merged and released, remove this and VolumeInfoList + export interface VolumeInspectInfo { + Name: string; + Driver: string; + Mountpoint: string; + Status?: { [key: string]: string }; + Labels: { [key: string]: string }; + Scope: 'local' | 'global'; + // Field is always present, but sometimes is null + Options: { [key: string]: string } | null; + // Field is sometimes present, and sometimes null + UsageData?: { + Size: number; + RefCount: number; + } | null; + } + + export interface VolumeInfoList { + Volumes: Dockerode.VolumeInspectInfo[]; + Warnings: string[]; + } }