Change config function providers to be mutable

Also change logsChannelSecret value to be queried with the api backend,
so that logs are not shared between instances. This has been implemented
as the first config function provider with mutability.

Change-type: minor
Closes: #675
Signed-off-by: Cameron Diver <cameron@resin.io>

src/config.ts

@@ -6,137 +6,22 @@ import { generateUniqueKey } from 'resin-register-device';
 
 import ConfigJsonConfigBackend from './config/configJson';
 
+import { ConfigProviderFunctions, createProviderFunctions } from './config/functions';
 import * as constants from './lib/constants';
-import * as osRelease from './lib/os-release';
 import { ConfigMap, ConfigSchema, ConfigValue } from './lib/types';
 
 import DB = require('./db');
-import supervisorVersion = require('./lib/supervisor-version');
 
 interface ConfigOpts {
 	db: DB;
 	configPath: string;
 }
 
-// A provider for schema entries with source 'func'
-type ConfigFunction = (...args: any[]) => Bluebird<any>;
-
 class Config extends EventEmitter {
 
 	private db: DB;
 	private configJsonBackend: ConfigJsonConfigBackend;
-
+	private providerFunctions: ConfigProviderFunctions;
-	private funcs: { [functionName: string]: ConfigFunction } = {
-		version: () => {
-			return Bluebird.resolve(supervisorVersion);
-		},
-		currentApiKey: () => {
-			return this.getMany([ 'apiKey', 'deviceApiKey' ])
-				.then(({ apiKey, deviceApiKey }) => {
-					return apiKey || deviceApiKey;
-				});
-		},
-		offlineMode: () => {
-			return this.getMany([ 'resinApiEndpoint', 'supervisorOfflineMode' ])
-				.then(({ resinApiEndpoint, supervisorOfflineMode }) => {
-					return Boolean(supervisorOfflineMode) || !Boolean(resinApiEndpoint);
-				});
-		},
-		pubnub: () => {
-			return this.getMany([ 'pubnubSubscribeKey', 'pubnubPublishKey' ])
-				.then(({ pubnubSubscribeKey, pubnubPublishKey }) => {
-					return {
-						subscribe_key: pubnubSubscribeKey,
-						publish_key: pubnubPublishKey,
-						ssl: true,
-					};
-				});
-		},
-		resinApiEndpoint: () => {
-			// Fallback to checking if an API endpoint was passed via env vars if there's none
-			// in config.json (legacy)
-			return this.get('apiEndpoint')
-				.then((apiEndpoint) => {
-					return apiEndpoint || constants.apiEndpointFromEnv;
-				});
-		},
-		provisioned: () => {
-			return this.getMany([
-				'uuid',
-				'resinApiEndpoint',
-				'registered_at',
-				'deviceId',
-			])
-				.then((requiredValues) => {
-					return _.every(_.values(requiredValues), Boolean);
-				});
-		},
-		osVersion: () => {
-			return osRelease.getOSVersion(constants.hostOSVersionPath);
-		},
-		osVariant: () => {
-			return osRelease.getOSVariant(constants.hostOSVersionPath);
-		},
-		provisioningOptions: () => {
-			return this.getMany([
-				'uuid',
-				'userId',
-				'applicationId',
-				'apiKey',
-				'deviceApiKey',
-				'deviceType',
-				'resinApiEndpoint',
-				'apiTimeout',
-				'registered_at',
-				'deviceId',
-			]).then((conf) => {
-				return {
-					uuid: conf.uuid,
-					applicationId: conf.applicationId,
-					userId: conf.userId,
-					deviceType: conf.deviceType,
-					provisioningApiKey: conf.apiKey,
-					deviceApiKey: conf.deviceApiKey,
-					apiEndpoint: conf.resinApiEndpoint,
-					apiTimeout: conf.apiTimeout,
-					registered_at: conf.registered_at,
-					deviceId: conf.deviceId,
-				};
-			});
-		},
-		mixpanelHost: () => {
-			return this.get('resinApiEndpoint')
-				.then((apiEndpoint) => {
-					return `${apiEndpoint}/mixpanel`;
-				});
-		},
-		extendedEnvOptions: () => {
-			return this.getMany([
-				'uuid',
-				'listenPort',
-				'name',
-				'apiSecret',
-				'deviceApiKey',
-				'version',
-				'deviceType',
-				'osVersion',
-			]);
-		},
-		fetchOptions: () => {
-			return this.getMany([
-				'uuid',
-				'currentApiKey',
-				'resinApiEndpoint',
-				'deltaEndpoint',
-				'delta',
-				'deltaRequestTimeout',
-				'deltaApplyTimeout',
-				'deltaRetryCount',
-				'deltaRetryInterval',
-				'deltaVersion',
-			]);
-		},
-	};
 
 	public schema: ConfigSchema = {
 		apiEndpoint: { source: 'config.json' },
@@ -175,7 +60,6 @@ class Config extends EventEmitter {
 
 		// NOTE: all 'db' values are stored and loaded as *strings*,
 		apiSecret: { source: 'db', mutable: true },
-		logsChannelSecret: { source: 'db', mutable: true },
 		name: { source: 'db', mutable: true },
 		initialConfigReported: { source: 'db', mutable: true, default: 'false' },
 		initialConfigSaved: { source: 'db', mutable: true, default: 'false' },
@@ -195,12 +79,16 @@ class Config extends EventEmitter {
 		// a JSON value, which is either null, or { app: number, commit: string }
 		pinDevice: { source: 'db', mutable: true, default: 'null' },
 		currentCommit: { source: 'db', mutable: true },
+
+		// Mutable functions, defined in mutableFuncs
+		logsChannelSecret: { source: 'func', mutable: true },
 	};
 
 	public constructor({ db, configPath }: ConfigOpts) {
 		super();
 		this.db = db;
 		this.configJsonBackend = new ConfigJsonConfigBackend(this.schema, configPath);
+		this.providerFunctions = createProviderFunctions(this, db);
 	}
 
 	public init(): Bluebird<void> {
@@ -219,7 +107,7 @@ class Config extends EventEmitter {
 			}
 			switch(this.schema[key].source) {
 				case 'func':
-					return this.funcs[key]()
+					return this.providerFunctions[key].get()
 						.catch((e) => {
 							console.error(`Error getting config value for ${key}`, e, e.stack);
 							return null;
@@ -254,9 +142,10 @@ class Config extends EventEmitter {
 
 	public set(keyValues: ConfigMap, trx?: Transaction): Bluebird<void> {
 		return Bluebird.try(() => {
-			// Split the values into database and configJson values
+
-			type SplitConfigBackend = { configJsonVals: ConfigMap, dbVals: ConfigMap };
+			// Split the values based on which storage backend they use
-			const { configJsonVals, dbVals }: SplitConfigBackend = _.reduce(keyValues, (acc: SplitConfigBackend, val, key) => {
+			type SplitConfigBackend = { configJsonVals: ConfigMap, dbVals: ConfigMap, fnVals: ConfigMap };
+			const { configJsonVals, dbVals, fnVals }: SplitConfigBackend = _.reduce(keyValues, (acc: SplitConfigBackend, val, key) => {
 				if (this.schema[key] == null || !this.schema[key].mutable) {
 					throw new Error(`Config field ${key} not found or is immutable in config.set`);
 				}
@@ -264,12 +153,15 @@ class Config extends EventEmitter {
 					acc.configJsonVals[key] = val;
 				} else if (this.schema[key].source === 'db') {
 					acc.dbVals[key] = val;
+				} else if (this.schema[key].source === 'func') {
+					acc.fnVals[key] = val;
 				} else {
 					throw new Error(`Unknown config backend for key: ${key}, backend: ${this.schema[key].source}`);
 				}
 				return acc;
-			}, { configJsonVals: { }, dbVals: { } });
+			}, { configJsonVals: { }, dbVals: { }, fnVals: { } });
 
+			// Set these values, taking into account the knex transaction
 			const setValuesInTransaction = (tx: Transaction): Bluebird<void> => {
 				const dbKeys = _.keys(dbVals);
 				return this.getMany(dbKeys, tx)
@@ -281,6 +173,15 @@ class Config extends EventEmitter {
 							}
 						});
 					})
+					.then(() => {
+						return Bluebird.map(_.toPairs(fnVals), ([key, value]) => {
+							const fn = this.providerFunctions[key];
+							if (fn.set == null) {
+								throw new Error(`Attempting to set provider function without set() method implemented - key: ${key}`);
+							}
+							return fn.set(value, tx);
+						});
+					})
 					.then(() => {
 						if (!_.isEmpty(configJsonVals)) {
 							return this.configJsonBackend.set(configJsonVals);
@@ -313,6 +214,15 @@ class Config extends EventEmitter {
 				return this.configJsonBackend.remove(key);
 			} else if (this.schema[key].source === 'db') {
 				return this.db.models('config').del().where({ key });
+			} else if (this.schema[key].source === 'func') {
+				const mutFn = this.providerFunctions[key];
+				if (mutFn == null) {
+					throw new Error(`Could not find provider function for config ${key}!`);
+				}
+				if (mutFn.remove == null) {
+					throw new Error(`Could not find removal provider function for config ${key}`);
+				}
+				return mutFn.remove();
 			} else {
 				throw new Error(`Unknown or unsupported config backend: ${this.schema[key].source}`);
 			}

src/config/functions.ts

@@ -0,0 +1,197 @@
+import * as Bluebird from 'bluebird';
+import { Transaction } from 'knex';
+import * as _ from 'lodash';
+
+import Config = require('../config');
+import DB = require('../db');
+import supervisorVersion = require('../lib/supervisor-version');
+
+import * as constants from '../lib/constants';
+import * as osRelease from '../lib/os-release';
+import { ConfigValue } from '../lib/types';
+
+// A provider for schema entries with source 'func'
+type ConfigProviderFunctionGetter = () => Bluebird<any>;
+type ConfigProviderFunctionSetter = (value: ConfigValue, tx?: Transaction) => Bluebird<void>;
+type ConfigProviderFunctionRemover = () => Bluebird<void>;
+
+interface ConfigProviderFunction {
+	get: ConfigProviderFunctionGetter;
+	set?: ConfigProviderFunctionSetter;
+	remove?: ConfigProviderFunctionRemover;
+}
+
+export interface ConfigProviderFunctions {
+	[key: string]: ConfigProviderFunction;
+}
+
+export function createProviderFunctions(config: Config, db: DB): ConfigProviderFunctions {
+	return {
+		logsChannelSecret: {
+			get: () => {
+				// Return the logsChannelSecret which corresponds to the current backend
+				return config.get('apiEndpoint')
+					.then((backend = '') => {
+						return db.models('logsChannelSecret').select('secret').where({ backend });
+					})
+					.then(([ conf ]) => {
+						if (conf != null) {
+							return conf.secret;
+						}
+						return;
+					});
+			},
+			set: (value: string, tx?: Transaction) => {
+				// Store the secret with the current backend
+				return config.get('apiEndpoint')
+					.then((backend: string) => {
+						return db.upsertModel(
+							'logsChannelSecret',
+							{ backend: backend || '', secret: value },
+							{ backend: backend || '' },
+							tx,
+						);
+					});
+			},
+			remove: () => {
+				return config.get('apiEndpoint')
+					.then((backend) => {
+						return db.models('logsChannelSecret').where({ backend: backend || '' }).del();
+					});
+			},
+		},
+		version: {
+			get: () => {
+				return Bluebird.resolve(supervisorVersion);
+			},
+		},
+		currentApiKey: {
+			get: () => {
+				return config.getMany([ 'apiKey', 'deviceApiKey' ])
+					.then(({ apiKey, deviceApiKey }) => {
+						return apiKey || deviceApiKey;
+					});
+			},
+		},
+		offlineMode: {
+			get: () => {
+				return config.getMany([ 'resinApiEndpoint', 'supervisorOfflineMode' ])
+					.then(({ resinApiEndpoint, supervisorOfflineMode }) => {
+						return Boolean(supervisorOfflineMode) || !Boolean(resinApiEndpoint);
+					});
+			},
+		},
+		pubnub: {
+			get: () => {
+				return config.getMany([ 'pubnubSubscribeKey', 'pubnubPublishKey' ])
+					.then(({ pubnubSubscribeKey, pubnubPublishKey }) => {
+						return {
+							subscribe_key: pubnubSubscribeKey,
+							publish_key: pubnubPublishKey,
+							ssl: true,
+						};
+					});
+			},
+		},
+		resinApiEndpoint: {
+			get: () => {
+				// Fallback to checking if an API endpoint was passed via env vars if there's none
+				// in config.json (legacy)
+				return config.get('apiEndpoint')
+					.then((apiEndpoint) => {
+						return apiEndpoint || (constants.apiEndpointFromEnv || '');
+					});
+			},
+		},
+		provisioned: {
+			get: () => {
+				return config.getMany([
+					'uuid',
+					'resinApiEndpoint',
+					'registered_at',
+					'deviceId',
+				])
+					.then((requiredValues) => {
+						return _.every(_.values(requiredValues), Boolean);
+					});
+			},
+		},
+		osVersion: {
+			get: () => {
+				return osRelease.getOSVersion(constants.hostOSVersionPath);
+			},
+		},
+		osVariant: {
+			get: () => {
+				return osRelease.getOSVariant(constants.hostOSVersionPath);
+			},
+		},
+		provisioningOptions: {
+			get: () => {
+				return config.getMany([
+					'uuid',
+					'userId',
+					'applicationId',
+					'apiKey',
+					'deviceApiKey',
+					'deviceType',
+					'resinApiEndpoint',
+					'apiTimeout',
+					'registered_at',
+					'deviceId',
+				]).then((conf) => {
+					return {
+						uuid: conf.uuid,
+						applicationId: conf.applicationId,
+						userId: conf.userId,
+						deviceType: conf.deviceType,
+						provisioningApiKey: conf.apiKey,
+						deviceApiKey: conf.deviceApiKey,
+						apiEndpoint: conf.resinApiEndpoint,
+						apiTimeout: conf.apiTimeout,
+						registered_at: conf.registered_at,
+						deviceId: conf.deviceId,
+					};
+				});
+			},
+		},
+		mixpanelHost: {
+			get: () => {
+				return config.get('resinApiEndpoint')
+					.then((apiEndpoint) => {
+						return `${apiEndpoint}/mixpanel`;
+					});
+			},
+		},
+		extendedEnvOptions: {
+			get: () => {
+				return config.getMany([
+					'uuid',
+					'listenPort',
+					'name',
+					'apiSecret',
+					'deviceApiKey',
+					'version',
+					'deviceType',
+					'osVersion',
+				]);
+			},
+		},
+		fetchOptions: {
+			get: () => {
+				return config.getMany([
+					'uuid',
+					'currentApiKey',
+					'resinApiEndpoint',
+					'deltaEndpoint',
+					'delta',
+					'deltaRequestTimeout',
+					'deltaApplyTimeout',
+					'deltaRetryCount',
+					'deltaRetryInterval',
+					'deltaVersion',
+				]);
+			},
+		},
+	};
+}

src/migrations/20181907164000-endpoint-specific-logschannel.js

@@ -0,0 +1,57 @@
+const fs = require('fs');
+const configJsonPath = process.env.CONFIG_MOUNT_POINT;
+
+exports.up = function (knex, Promise) {
+	return new Promise((resolve, reject) => {
+		if (!configJsonPath) {
+			console.log('Unable to locate config.json! Things may fail unexpectedly!');
+			resolve({});
+			return;
+		}
+		fs.readFile(configJsonPath, (err, data) => {
+			if (err) {
+				console.log('Failed to read config.json! Things may fail unexpectedly!');
+				resolve({});
+				return;
+			}
+			try {
+				const parsed = JSON.parse(data.toString());
+				resolve(parsed);
+			} catch (e) {
+				console.log('Failed to parse config.json! Things may fail unexpectedly!');
+				resolve({});
+			}
+		});
+	})
+		.tap(() => {
+			// take the logsChannelSecret, and the apiEndpoint config field,
+			// and store them in a new table
+			return knex.schema.createTableIfNotExists('logsChannelSecret', (t) => {
+				t.string('backend');
+				t.string('secret');
+			});
+		})
+		.then((config) => {
+			return knex('config').where({ key: 'logsChannelSecret' }).select('value')
+				.then((results) => {
+					if (results.length === 0) {
+						return { config, secret: null };
+					}
+					return { config, secret: results[0].value };
+				});
+		})
+		.then(({ config, secret }) => {
+			return knex('logsChannelSecret').insert({
+				backend: config.apiEndpoint || '',
+				secret
+			});
+		})
+		.then(() => {
+			return knex('config').where('key', 'logsChannelSecret').del();
+		});
+
+}
+
+exports.down = function (knex, Promise) {
+	return Promise.reject(new Error('Not Implemented'));
+}

test/03-config.spec.coffee

@@ -114,3 +114,20 @@ describe 'Config', ->
 		.then (osVariant) ->
 			constants.hostOSVersionPath = oldPath
 			expect(osVariant).to.be.undefined
+
+	describe.only 'Function config providers', ->
+		it 'should allow setting of mutable function config options', ->
+			@conf.set({ logsChannelSecret: 'test' })
+			.then =>
+				expect(@conf.get('logsChannelSecret')).to.eventually.equal('test')
+
+		it 'should allow removing of mutabe function config options', ->
+			@conf.remove('logsChannelSecret')
+			.then =>
+				expect(@conf.get('logsChannelSecret')).to.eventually.be.undefined
+
+		it 'should throw if a non-mutable function provider is set', ->
+			expect(@conf.set({ version: 'some-version' })).to.be.rejected
+
+		it 'should throw if a non-mutbale function provider is removed', ->
+			expect(@conf.remove('version')).to.be.rejected