Refactor device-config to support configuring multiple backends

Signed-off-by: Miguel Casqueira <miguel@balena.io>

src/config/backends/index.ts

@@ -0,0 +1,11 @@
+import { Extlinux } from './extlinux';
+import { ExtraUEnv } from './extra-uEnv';
+import { ConfigTxt } from './config-txt';
+import { ConfigFs } from './config-fs';
+
+export default [
+	new Extlinux(),
+	new ExtraUEnv(),
+	new ConfigTxt(),
+	new ConfigFs(),
+];

src/config/utils.ts

@@ -1,41 +1,23 @@
 import * as _ from 'lodash';
+import * as Bluebird from 'bluebird';
 
+import * as config from '../config';
 import * as constants from '../lib/constants';
 import { getMetaOSRelease } from '../lib/os-release';
 import { EnvVarObject } from '../lib/types';
-import { Extlinux } from './backends/extlinux';
-import { ExtraUEnv } from './backends/extra-uEnv';
-import { ConfigTxt } from './backends/config-txt';
-import { ConfigFs } from './backends/config-fs';
+import Backends from './backends';
 import { ConfigOptions, ConfigBackend } from './backends/backend';
 
-const configBackends = [
-	new Extlinux(),
-	new ExtraUEnv(),
-	new ConfigTxt(),
-	new ConfigFs(),
-];
-
-export const initialiseConfigBackend = async (deviceType: string) => {
-	const backend = await getConfigBackend(deviceType);
-	if (backend) {
-		await backend.initialise();
-		return backend;
-	}
-};
-
-async function getConfigBackend(
-	deviceType: string,
-): Promise<ConfigBackend | undefined> {
-	// Some backends are only supported by certain release versions so pass in metaRelease
-	const metaRelease = await getMetaOSRelease(constants.hostOSVersionPath);
-	let matched;
-	for (const backend of configBackends) {
-		if (await backend.matches(deviceType, metaRelease)) {
-			matched = backend;
-		}
-	}
-	return matched;
+export async function getSupportedBackends(): Promise<ConfigBackend[]> {
+	// Get required information to find supported backends
+	const [deviceType, metaRelease] = await Promise.all([
+		config.get('deviceType'),
+		getMetaOSRelease(constants.hostOSVersionPath),
+	]);
+	// Return list of configurable backends that match this deviceType and metaRelease
+	return Bluebird.filter(Backends, (backend: ConfigBackend) =>
+		backend.matches(deviceType, metaRelease),
+	);
 }
 
 export function envToBootConfig(
@@ -45,7 +27,6 @@ export function envToBootConfig(
 	if (configBackend == null) {
 		return {};
 	}
-
 	return _(env)
 		.pickBy((_val, key) => configBackend.isBootConfigVar(key))
 		.mapKeys((_val, key) => configBackend.processConfigVarName(key))
@@ -57,9 +38,9 @@ export function envToBootConfig(
 
 export function bootConfigToEnv(
 	configBackend: ConfigBackend,
-	config: ConfigOptions,
+	configOptions: ConfigOptions,
 ): EnvVarObject {
-	return _(config)
+	return _(configOptions)
 		.mapKeys((_val, key) => configBackend.createConfigVarName(key))
 		.mapValues((val) => {
 			if (_.isArray(val)) {
@@ -70,20 +51,6 @@ export function bootConfigToEnv(
 		.value();
 }
 
-function filterNamespaceFromConfig(
-	namespace: RegExp,
-	conf: { [key: string]: any },
-): { [key: string]: any } {
-	return _.mapKeys(
-		_.pickBy(conf, (_v, k) => {
-			return namespace.test(k);
-		}),
-		(_v, k) => {
-			return k.replace(namespace, '$1');
-		},
-	);
-}
-
 export function formatConfigKeys(
 	configBackend: ConfigBackend | null,
 	allowedKeys: string[],
@@ -113,3 +80,17 @@ export function formatConfigKeys(
 		);
 	});
 }
+
+function filterNamespaceFromConfig(
+	namespace: RegExp,
+	conf: { [key: string]: any },
+): { [key: string]: any } {
+	return _.mapKeys(
+		_.pickBy(conf, (_v, k) => {
+			return namespace.test(k);
+		}),
+		(_v, k) => {
+			return k.replace(namespace, '$1');
+		},
+	);
+}

src/device-config.ts

@@ -100,15 +100,17 @@ const actionExecutors: DeviceActionExecutors = {
 		}
 	},
 	setBootConfig: async (step) => {
-		const $configBackend = await getConfigBackend();
 		if (!_.isObject(step.target)) {
 			throw new Error('Non-dictionary passed to DeviceConfig.setBootConfig');
 		}
-		await setBootConfig($configBackend, step.target as Dictionary<string>);
+		const backends = await getConfigBackends();
+		for (const backend of backends) {
+			await setBootConfig(backend, step.target as Dictionary<string>);
+		}
 	},
 };
 
-let configBackend: ConfigBackend | null = null;
+const configBackends: ConfigBackend[] = [];
 
 const configKeys: Dictionary<ConfigOption> = {
 	appUpdatePollInterval: {
@@ -206,14 +208,19 @@ const rateLimits: Dictionary<{
 	},
 };
 
-async function getConfigBackend() {
-	if (configBackend != null) {
-		return configBackend;
+async function getConfigBackends(): Promise<ConfigBackend[]> {
+	// Exit early if we already have a list
+	if (configBackends.length > 0) {
+		return configBackends;
 	}
-	const dt = await config.get('deviceType');
-	configBackend = (await configUtils.initialiseConfigBackend(dt)) ?? null;
-
-	return configBackend;
+	// Get all the configurable backends this device supports
+	const backends = await configUtils.getSupportedBackends();
+	// Initialize each backend
+	for (const backend of backends) {
+		await backend.initialise();
+	}
+	// Return list of initialized ConfigBackends
+	return backends;
 }
 
 export async function setTarget(
@@ -264,37 +271,45 @@ export async function getTarget({
 	return conf;
 }
 
-export async function getCurrent() {
+export async function getCurrent(): Promise<Dictionary<string>> {
+	// Build a Dictionary of currently set config values
+	const currentConf: Dictionary<string> = {};
+	// Get environment variables
 	const conf = await config.getMany(
 		['deviceType'].concat(_.keys(configKeys)) as SchemaTypeKey[],
 	);
-
-	const $configBackend = await getConfigBackend();
-
-	const [vpnStatus, bootConfig] = await Promise.all([
-		getVPNEnabled(),
-		getBootConfig($configBackend),
-	]);
-
-	const currentConf: Dictionary<string> = {
-		// TODO: Fix this mess of half strings half boolean values everywhere
-		SUPERVISOR_VPN_CONTROL: vpnStatus != null ? vpnStatus.toString() : 'true',
-	};
-
+	// Add each value
 	for (const key of _.keys(configKeys)) {
 		const { envVarName } = configKeys[key];
 		const confValue = conf[key as SchemaTypeKey];
 		currentConf[envVarName] = confValue != null ? confValue.toString() : '';
 	}
-
-	return _.assign(currentConf, bootConfig);
+	// Add VPN information
+	currentConf['SUPERVISOR_VPN_CONTROL'] = (await isVPNEnabled())
+		? 'true'
+		: 'false';
+	// Get list of configurable backends
+	const backends = await getConfigBackends();
+	// Add each backends configurable values
+	for (const backend of backends) {
+		_.assign(currentConf, await getBootConfig(backend));
+	}
+	// Return compiled configuration
+	return currentConf;
 }
 
 export async function formatConfigKeys(
 	conf: Dictionary<string>,
 ): Promise<Dictionary<any>> {
-	const backend = await getConfigBackend();
-	return configUtils.formatConfigKeys(backend, validKeys, conf);
+	const backends = await getConfigBackends();
+	const formattedKeys: Dictionary<any> = {};
+	for (const backend of backends) {
+		_.assign(
+			formattedKeys,
+			configUtils.formatConfigKeys(backend, validKeys, conf),
+		);
+	}
+	return formattedKeys;
 }
 
 export function getDefaults() {
@@ -314,16 +329,13 @@ export function resetRateLimits() {
 
 // Exported for tests
 export function bootConfigChangeRequired(
-	$configBackend: ConfigBackend | null,
+	configBackend: ConfigBackend | null,
 	current: Dictionary<string>,
 	target: Dictionary<string>,
 	deviceType: string,
 ): boolean {
-	const targetBootConfig = configUtils.envToBootConfig($configBackend, target);
-	const currentBootConfig = configUtils.envToBootConfig(
-		$configBackend,
-		current,
-	);
+	const targetBootConfig = configUtils.envToBootConfig(configBackend, target);
+	const currentBootConfig = configUtils.envToBootConfig(configBackend, current);
 
 	// Some devices require specific overlays, here we apply them
 	ensureRequiredOverlay(deviceType, targetBootConfig);
@@ -331,7 +343,7 @@ export function bootConfigChangeRequired(
 	if (!_.isEqual(currentBootConfig, targetBootConfig)) {
 		_.each(targetBootConfig, (value, key) => {
 			// Ignore null check because we can't get here if configBackend is null
-			if (!$configBackend!.isSupportedConfig(key)) {
+			if (!configBackend!.isSupportedConfig(key)) {
 				if (currentBootConfig[key] !== value) {
 					const err = `Attempt to change blacklisted config value ${key}`;
 					logger.logSystemMessage(
@@ -369,7 +381,6 @@ export async function getRequiredSteps(
 		'deviceType',
 		'unmanaged',
 	]);
-	const backend = await getConfigBackend();
 
 	const configChanges: Dictionary<string> = {};
 	const humanReadableConfigChanges: Dictionary<string> = {};
@@ -455,13 +466,16 @@ export async function getRequiredSteps(
 		return step;
 	});
 
-	// Do we need to change the boot config?
-	if (bootConfigChangeRequired(backend, current, target, deviceType)) {
-		steps.push({
-			action: 'setBootConfig',
-			target,
-		});
-	}
+	const backends = await getConfigBackends();
+	// Check for required bootConfig changes
+	backends.forEach((backend) => {
+		if (bootConfigChangeRequired(backend, current, target, deviceType)) {
+			steps.push({
+				action: 'setBootConfig',
+				target,
+			});
+		}
+	});
 
 	// Check if there is either no steps, or they are all
 	// noops, and we need to reboot. We want to do this
@@ -539,7 +553,7 @@ export async function setBootConfig(
 	}
 }
 
-async function getVPNEnabled(): Promise<boolean> {
+async function isVPNEnabled(): Promise<boolean> {
 	try {
 		const activeState = await dbus.serviceActiveState(vpnServiceName);
 		return !_.includes(['inactive', 'deactivating'], activeState);

test/17-config-utils.spec.ts

@@ -1,25 +1,124 @@
-import { expect } from './lib/chai-config';
-import * as configUtils from '../src/config/utils';
-import { ConfigTxt } from '../src/config/backends/config-txt';
+import { stub } from 'sinon';
+import * as _ from 'lodash';
 
-const configTxtBackend = new ConfigTxt();
+import { expect } from './lib/chai-config';
+import * as config from '../src/config';
+import { validKeys } from '../src/device-config';
+import * as configUtils from '../src/config/utils';
+import { ExtraUEnv } from '../src/config/backends/extra-uEnv';
+import { Extlinux } from '../src/config/backends/extlinux';
+import { ConfigTxt } from '../src/config/backends/config-txt';
+import { ConfigFs } from '../src/config/backends/config-fs';
+import { ConfigBackend } from '../src/config/backends/backend';
 
 describe('Config Utilities', () => {
-	describe('Boot config', () => {
-		it('correctly transforms environments to boot config objects', () => {
-			const bootConfig = configUtils.envToBootConfig(configTxtBackend, {
-				HOST_CONFIG_initramfs: 'initramf.gz 0x00800000',
-				HOST_CONFIG_dtparam: '"i2c=on","audio=on"',
-				HOST_CONFIG_dtoverlay:
-					'"ads7846","lirc-rpi,gpio_out_pin=17,gpio_in_pin=13"',
-				HOST_CONFIG_foobar: 'baz',
-			});
-			expect(bootConfig).to.deep.equal({
-				initramfs: 'initramf.gz 0x00800000',
-				dtparam: ['i2c=on', 'audio=on'],
-				dtoverlay: ['ads7846', 'lirc-rpi,gpio_out_pin=17,gpio_in_pin=13'],
-				foobar: 'baz',
-			});
+	it('gets list of supported backends', async () => {
+		// Stub so that we get an array containing only config-txt backend
+		const configStub = stub(config, 'get').resolves('raspberry');
+		// Get list of backends
+		const devices = await configUtils.getSupportedBackends();
+		expect(devices.length).to.equal(1);
+		expect(devices[0].constructor.name).to.equal('ConfigTxt');
+		// Restore stub
+		configStub.restore();
+		// TO-DO: When we have a device that will match for multiple backends
+		// add a test that we get more then 1 backend for that device
+	});
+
+	it('transforms environment variables to boot configs', () => {
+		_.forEach(CONFIGS, (configObj: any, key: string) => {
+			expect(
+				configUtils.envToBootConfig(BACKENDS[key], configObj.envVars),
+			).to.deep.equal(configObj.bootConfig);
+		});
+	});
+
+	it('transforms boot configs to environment variables', () => {
+		_.forEach(CONFIGS, (configObj: any, key: string) => {
+			expect(
+				configUtils.bootConfigToEnv(BACKENDS[key], configObj.bootConfig),
+			).to.deep.equal(configObj.envVars);
+		});
+	});
+
+	it('formats keys from config', () => {
+		// Pick any backend to use for test
+		// note: some of the values used will be specific to this backend
+		const backend = BACKENDS['extlinux'];
+		const formattedKeys = configUtils.formatConfigKeys(backend, validKeys, {
+			FOO: 'bar',
+			BAR: 'baz',
+			RESIN_HOST_CONFIG_foo: 'foobaz',
+			BALENA_HOST_CONFIG_foo: 'foobar',
+			RESIN_HOST_CONFIG_other: 'val',
+			BALENA_HOST_CONFIG_baz: 'bad',
+			BALENA_SUPERVISOR_POLL_INTERVAL: '100', // any device
+			BALENA_HOST_EXTLINUX_isolcpus: '1,2,3', // specific to backend
+			RESIN_HOST_EXTLINUX_fdt: '/boot/mycustomdtb.dtb', // specific to backend
+		});
+		expect(formattedKeys).to.deep.equal({
+			HOST_EXTLINUX_isolcpus: '1,2,3',
+			HOST_EXTLINUX_fdt: '/boot/mycustomdtb.dtb',
+			SUPERVISOR_POLL_INTERVAL: '100',
 		});
 	});
 });
+
+const BACKENDS: Record<string, ConfigBackend> = {
+	extraUEnv: new ExtraUEnv(),
+	extlinux: new Extlinux(),
+	configtxt: new ConfigTxt(),
+	configfs: new ConfigFs(),
+};
+
+const CONFIGS = {
+	extraUEnv: {
+		envVars: {
+			HOST_EXTLINUX_fdt: '/boot/mycustomdtb.dtb',
+			HOST_EXTLINUX_isolcpus: '1,2,3',
+			HOST_EXTLINUX_rootwait: '',
+		},
+		bootConfig: {
+			fdt: '/boot/mycustomdtb.dtb',
+			isolcpus: '1,2,3',
+			rootwait: '',
+		},
+	},
+	extlinux: {
+		envVars: {
+			HOST_EXTLINUX_fdt: '/boot/mycustomdtb.dtb',
+			HOST_EXTLINUX_isolcpus: '1,2,3',
+			HOST_EXTLINUX_rootwait: '',
+		},
+		bootConfig: {
+			fdt: '/boot/mycustomdtb.dtb',
+			isolcpus: '1,2,3',
+			rootwait: '',
+		},
+	},
+	configtxt: {
+		envVars: {
+			HOST_CONFIG_initramfs: 'initramf.gz 0x00800000',
+			HOST_CONFIG_dtparam: '"i2c=on","audio=on"',
+			HOST_CONFIG_dtoverlay:
+				'"ads7846","lirc-rpi,gpio_out_pin=17,gpio_in_pin=13"',
+			HOST_CONFIG_foobar: 'baz',
+		},
+		bootConfig: {
+			initramfs: 'initramf.gz 0x00800000',
+			dtparam: ['i2c=on', 'audio=on'],
+			dtoverlay: ['ads7846', 'lirc-rpi,gpio_out_pin=17,gpio_in_pin=13'],
+			foobar: 'baz',
+		},
+	},
+	// TO-DO: Config-FS is commented out because it behaves differently and doesn't
+	// add value to the Config Utilities if we make it work but would like to add it
+	// configfs: {
+	// 	envVars: {
+	// 		ssdt: 'spidev1,1'
+	// 	},
+	// 	bootConfig: {
+	// 		ssdt: ['spidev1,1']
+	// 	},
+	// },
+};