From e00687408ce9c1b20217f25b1c906b7855aab83f Mon Sep 17 00:00:00 2001 From: Felipe Lalanne Date: Tue, 19 Jul 2022 12:29:13 -0400 Subject: [PATCH 1/3] Disable event tracking The supervisor used to rely on specific event reporting for identifying issues at runtime. As the platform has grown, it has become much more difficult to get any signal from the event noise. Recently the API side for these events has been disabled, meaning these events only contribute to bandwidth consumption. This commit disables the event reporting feature of the supervisor which will be most likely replaced by something like Sentry in the near future. Change-type: minor --- src/api-binder/index.ts | 1 - src/event-tracker.ts | 63 ------- src/lib/api-helper.ts | 1 - src/supervisor.ts | 2 - test/legacy/08-event-tracker.spec.ts | 248 --------------------------- 5 files changed, 315 deletions(-) delete mode 100644 test/legacy/08-event-tracker.spec.ts diff --git a/src/api-binder/index.ts b/src/api-binder/index.ts index e206ba9c..352082a2 100644 --- a/src/api-binder/index.ts +++ b/src/api-binder/index.ts @@ -546,7 +546,6 @@ export let balenaApi: PinejsClientRequest | null = null; export const initialized = _.once(async () => { await config.initialized(); - await eventTracker.initialized(); await deviceState.initialized(); const { unmanaged, apiEndpoint, currentApiKey } = await config.getMany([ diff --git a/src/event-tracker.ts b/src/event-tracker.ts index 1ee71ff0..79174bf5 100644 --- a/src/event-tracker.ts +++ b/src/event-tracker.ts @@ -1,18 +1,10 @@ import mask = require('json-mask'); import * as _ from 'lodash'; -import * as memoizee from 'memoizee'; -import * as mixpanel from 'mixpanel'; -import * as config from './config'; import log from './lib/supervisor-console'; -import supervisorVersion = require('./lib/supervisor-version'); export type EventTrackProperties = Dictionary; -// The minimum amount of time to wait between sending -// events of the same type -const eventDebounceTime = 60000; - const mixpanelMask = [ 'appId', 'delay', @@ -24,39 +16,10 @@ const mixpanelMask = [ 'stateDiff/local(os_version,supervisor_version,ip_address,apps/*/services)', ].join(','); -let defaultProperties: EventTrackProperties; -// We must export this for the tests, but we make no references -// to it within the rest of the supervisor codebase -export let client: mixpanel.Mixpanel | null = null; - -export const initialized = _.once(async () => { - await config.initialized(); - - const { unmanaged, mixpanelHost, mixpanelToken, uuid } = await config.getMany( - ['unmanaged', 'mixpanelHost', 'mixpanelToken', 'uuid'], - ); - - defaultProperties = { - distinct_id: uuid, - uuid, - supervisorVersion, - }; - - if (unmanaged || mixpanelHost == null || mixpanelToken == null) { - return; - } - client = mixpanel.init(mixpanelToken, { - host: mixpanelHost.host, - path: mixpanelHost.path, - }); -}); - export async function track( event: string, properties: EventTrackProperties | Error = {}, ) { - await initialized(); - if (properties instanceof Error) { properties = { error: properties }; } @@ -73,30 +36,4 @@ export async function track( // Don't send potentially sensitive information, by using a whitelist properties = mask(properties, mixpanelMask); log.event('Event:', event, JSON.stringify(properties)); - if (client == null) { - return; - } - - properties = assignDefaultProperties(properties); - throttleddLogger(event)(properties); -} - -const throttleddLogger = memoizee( - (event: string) => { - // Call this function at maximum once every minute - return _.throttle( - (properties: EventTrackProperties | Error) => { - client?.track(event, properties); - }, - eventDebounceTime, - { leading: true }, - ); - }, - { primitive: true }, -); - -function assignDefaultProperties( - properties: EventTrackProperties, -): EventTrackProperties { - return _.merge({}, properties, defaultProperties); } diff --git a/src/lib/api-helper.ts b/src/lib/api-helper.ts index 352fd733..d073c19d 100644 --- a/src/lib/api-helper.ts +++ b/src/lib/api-helper.ts @@ -156,7 +156,6 @@ export const provision = async ( opts: KeyExchangeOpts, ) => { await config.initialized(); - await eventTracker.initialized(); let device: Device | null = null; diff --git a/src/supervisor.ts b/src/supervisor.ts index 38039234..30bbcdc3 100644 --- a/src/supervisor.ts +++ b/src/supervisor.ts @@ -2,7 +2,6 @@ import * as apiBinder from './api-binder'; import * as db from './db'; import * as config from './config'; import * as deviceState from './device-state'; -import * as eventTracker from './event-tracker'; import { intialiseContractRequirements } from './lib/contracts'; import { normaliseLegacyDatabase } from './lib/legacy'; import * as osRelease from './lib/os-release'; @@ -36,7 +35,6 @@ export class Supervisor { await db.initialized(); await config.initialized(); - await eventTracker.initialized(); await avahi.initialized(); log.debug('Starting logging infrastructure'); await logger.initialized(); diff --git a/test/legacy/08-event-tracker.spec.ts b/test/legacy/08-event-tracker.spec.ts deleted file mode 100644 index 902719df..00000000 --- a/test/legacy/08-event-tracker.spec.ts +++ /dev/null @@ -1,248 +0,0 @@ -import { SinonStub, stub, spy, SinonSpy } from 'sinon'; -import { expect } from 'chai'; -import * as mixpanel from 'mixpanel'; - -import log from '~/lib/supervisor-console'; -import supervisorVersion = require('~/lib/supervisor-version'); -import * as config from '~/src/config'; - -describe('EventTracker', () => { - let logEventStub: SinonStub; - before(() => { - logEventStub = stub(log, 'event'); - - delete require.cache[require.resolve('~/src/event-tracker')]; - }); - - afterEach(() => { - logEventStub.reset(); - }); - - after(() => { - logEventStub.restore(); - }); - - describe('Unmanaged', () => { - let configStub: SinonStub; - let eventTracker: typeof import('~/src/event-tracker'); - - before(async () => { - configStub = stub(config, 'getMany').returns( - Promise.resolve({ - unmanaged: true, - uuid: 'foobar', - mixpanelHost: { host: '', path: '' }, - mixpanelToken: '', - }) as any, - ); - - eventTracker = await import('~/src/event-tracker'); - }); - - after(() => { - configStub.restore(); - - delete require.cache[require.resolve('~/src/event-tracker')]; - }); - - it('initializes in unmanaged mode', () => { - expect(eventTracker.initialized()).to.be.fulfilled.then(() => { - expect(eventTracker.client).to.be.null; - }); - }); - - it('logs events in unmanaged mode, with the correct properties', async () => { - await eventTracker.track('Test event', { appId: 'someValue' }); - expect(logEventStub).to.be.calledWith( - 'Event:', - 'Test event', - JSON.stringify({ appId: 'someValue' }), - ); - }); - }); - - describe('Init', () => { - let eventTracker: typeof import('~/src/event-tracker'); - let configStub: SinonStub; - let mixpanelSpy: SinonSpy; - - before(async () => { - configStub = stub(config, 'getMany').returns( - Promise.resolve({ - mixpanelToken: 'someToken', - uuid: 'barbaz', - mixpanelHost: { host: '', path: '' }, - unmanaged: false, - }) as any, - ); - - mixpanelSpy = spy(mixpanel, 'init'); - - eventTracker = await import('~/src/event-tracker'); - }); - - after(() => { - configStub.restore(); - mixpanelSpy.restore(); - - delete require.cache[require.resolve('~/src/event-tracker')]; - }); - - it('initializes a mixpanel client when not in unmanaged mode', () => { - expect(eventTracker.initialized()).to.be.fulfilled.then(() => { - expect(mixpanel.init).to.have.been.calledWith('someToken'); - // @ts-expect-error - expect(eventTracker.client.token).to.equal('someToken'); - // @ts-expect-error - expect(eventTracker.client.track).to.be.a('function'); - }); - }); - }); - - describe('Managed', () => { - let eventTracker: typeof import('~/src/event-tracker'); - let configStub: SinonStub; - let mixpanelStub: SinonStub; - - before(async () => { - configStub = stub(config, 'getMany').returns( - Promise.resolve({ - mixpanelToken: 'someToken', - uuid: 'barbaz', - mixpanelHost: { host: '', path: '' }, - unmanaged: false, - }) as any, - ); - - mixpanelStub = stub(mixpanel, 'init').returns({ - token: 'someToken', - track: stub(), - } as any); - - eventTracker = await import('~/src/event-tracker'); - await eventTracker.initialized(); - }); - - after(() => { - configStub.restore(); - mixpanelStub.restore(); - - delete require.cache[require.resolve('~/src/event-tracker')]; - }); - - it('calls the mixpanel client track function with the event, properties and uuid as distinct_id', async () => { - await eventTracker.track('Test event 2', { appId: 'someOtherValue' }); - - expect(logEventStub).to.be.calledWith( - 'Event:', - 'Test event 2', - JSON.stringify({ appId: 'someOtherValue' }), - ); - // @ts-expect-error - expect(eventTracker.client.track).to.be.calledWith('Test event 2', { - appId: 'someOtherValue', - uuid: 'barbaz', - distinct_id: 'barbaz', - supervisorVersion, - }); - }); - - it('can be passed an Error and it is added to the event properties', async () => { - const theError = new Error('something went wrong'); - await eventTracker.track('Error event', theError); - // @ts-expect-error - expect(eventTracker.client.track).to.be.calledWith('Error event', { - error: { - message: theError.message, - stack: theError.stack, - }, - uuid: 'barbaz', - distinct_id: 'barbaz', - supervisorVersion, - }); - }); - - it('hides service environment variables, to avoid logging keys or secrets', async () => { - const props = { - service: { - appId: '1', - environment: { - RESIN_API_KEY: 'foo', - RESIN_SUPERVISOR_API_KEY: 'bar', - OTHER_VAR: 'hi', - }, - }, - }; - await eventTracker.track('Some app event', props); - // @ts-expect-error - expect(eventTracker.client.track).to.be.calledWith('Some app event', { - service: { appId: '1' }, - uuid: 'barbaz', - distinct_id: 'barbaz', - supervisorVersion, - }); - }); - - it('should handle being passed no properties object', () => { - expect(eventTracker.track('no-options')).to.be.fulfilled; - }); - }); - - describe('Rate limiting', () => { - let eventTracker: typeof import('~/src/event-tracker'); - let mixpanelStub: SinonStub; - - before(async () => { - mixpanelStub = stub(mixpanel, 'init').returns({ - track: stub(), - } as any); - eventTracker = await import('~/src/event-tracker'); - await eventTracker.initialized(); - }); - - after(() => { - mixpanelStub.restore(); - - delete require.cache[require.resolve('~/src/event-tracker')]; - }); - - it('should rate limit events of the same type', async () => { - // @ts-expect-error resetting a non-stub typed function - eventTracker.client?.track.reset(); - - await eventTracker.track('test', {}); - await eventTracker.track('test', {}); - await eventTracker.track('test', {}); - await eventTracker.track('test', {}); - await eventTracker.track('test', {}); - - expect(eventTracker.client?.track).to.have.callCount(1); - }); - - it('should rate limit events of the same type with different arguments', async () => { - // @ts-expect-error resetting a non-stub typed function - eventTracker.client?.track.reset(); - - await eventTracker.track('test2', { a: 1 }); - await eventTracker.track('test2', { b: 2 }); - await eventTracker.track('test2', { c: 3 }); - await eventTracker.track('test2', { d: 4 }); - await eventTracker.track('test2', { e: 5 }); - - expect(eventTracker.client?.track).to.have.callCount(1); - }); - - it('should not rate limit events of different types', async () => { - // @ts-expect-error resetting a non-stub typed function - eventTracker.client?.track.reset(); - - await eventTracker.track('test3', { a: 1 }); - await eventTracker.track('test4', { b: 2 }); - await eventTracker.track('test5', { c: 3 }); - await eventTracker.track('test6', { d: 4 }); - await eventTracker.track('test7', { e: 5 }); - - expect(eventTracker.client?.track).to.have.callCount(5); - }); - }); -}); From b168cc35a03bebed22f8a97cdc3ae72e143d8ad0 Mon Sep 17 00:00:00 2001 From: Felipe Lalanne Date: Tue, 20 Sep 2022 11:16:58 -0300 Subject: [PATCH 2/3] Remove mixpanel configurations Mixpanel configurations and packages are no longer used. This removes deadcode from the supervisor. --- Dockerfile.template | 4 +- package-lock.json | 129 -------------------------------------- package.json | 1 - src/config/functions.ts | 10 --- src/config/schema-type.ts | 10 --- src/config/schema.ts | 5 -- src/lib/constants.ts | 1 - src/supervisor.ts | 2 - 8 files changed, 1 insertion(+), 161 deletions(-) diff --git a/Dockerfile.template b/Dockerfile.template index 9307cdea..e7e7aaa6 100644 --- a/Dockerfile.template +++ b/Dockerfile.template @@ -68,12 +68,10 @@ RUN apk add --no-cache \ ARG ARCH ARG VERSION=master -ARG DEFAULT_MIXPANEL_TOKEN=bananasbananas ENV CONFIG_MOUNT_POINT=/boot/config.json \ LED_FILE=/dev/null \ SUPERVISOR_IMAGE=balena/$ARCH-supervisor \ - VERSION=$VERSION \ - DEFAULT_MIXPANEL_TOKEN=$DEFAULT_MIXPANEL_TOKEN + VERSION=$VERSION ############################################################### # Use the base image to run integration tests and for livepush diff --git a/package-lock.json b/package-lock.json index 6df20ef4..d891784f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -78,7 +78,6 @@ "livepush": "^3.5.1", "lodash": "^4.17.21", "memoizee": "^0.4.14", - "mixpanel": "^0.10.3", "mocha": "^8.3.2", "mocha-pod": "^0.6.0", "mock-fs": "^4.14.0", @@ -2077,18 +2076,6 @@ "acorn": "^6.0.0 || ^7.0.0 || ^8.0.0" } }, - "node_modules/agent-base": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-4.3.0.tgz", - "integrity": "sha512-salcGninV0nPrwpGNn4VTXBb1SOuXQBiqbrNXoeizJsHrsL6ERFM2Ne3JUSBWRE6aeNJI2ROP/WEEIDUiDe3cg==", - "dev": true, - "dependencies": { - "es6-promisify": "^5.0.0" - }, - "engines": { - "node": ">= 4.0.0" - } - }, "node_modules/agentkeepalive": { "version": "4.2.1", "resolved": "https://registry.npmjs.org/agentkeepalive/-/agentkeepalive-4.2.1.tgz", @@ -5381,21 +5368,6 @@ "es6-symbol": "^3.1.1" } }, - "node_modules/es6-promise": { - "version": "4.2.8", - "resolved": "https://registry.npmjs.org/es6-promise/-/es6-promise-4.2.8.tgz", - "integrity": "sha512-HJDGx5daxeIvxdBxvG2cb9g4tEvwIk3i8+nhX0yGrYmZUzbkdg8QbDevheDB8gd0//uPj4c1EQua8Q+MViT0/w==", - "dev": true - }, - "node_modules/es6-promisify": { - "version": "5.0.0", - "resolved": "http://registry.npmjs.org/es6-promisify/-/es6-promisify-5.0.0.tgz", - "integrity": "sha1-UQnWLz5W6pZ8S2NQWu8IKRyKUgM=", - "dev": true, - "dependencies": { - "es6-promise": "^4.0.3" - } - }, "node_modules/es6-symbol": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/es6-symbol/-/es6-symbol-3.1.3.tgz", @@ -7498,35 +7470,6 @@ "integrity": "sha1-7AbBDgo0wPL68Zn3/X/Hj//QPHM=", "dev": true }, - "node_modules/https-proxy-agent": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-3.0.0.tgz", - "integrity": "sha512-y4jAxNEihqvBI5F3SaO2rtsjIOnnNA8sEbuiP+UhJZJHeM2NRm6c09ax2tgqme+SgUUvjao2fJXF4h3D6Cb2HQ==", - "dev": true, - "dependencies": { - "agent-base": "^4.3.0", - "debug": "^3.1.0" - }, - "engines": { - "node": ">= 4.5.0" - } - }, - "node_modules/https-proxy-agent/node_modules/debug": { - "version": "3.2.6", - "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.6.tgz", - "integrity": "sha512-mel+jf7nrtEl5Pn1Qx46zARXKDpBbvzezse7p7LqINmdoIk8PYP5SySaxEmYv6TZ0JyEKA1hsCId6DIhgITtWQ==", - "deprecated": "Debug versions >=3.2.0 <3.2.7 || >=4 <4.3.1 have a low-severity ReDos regression when used in a Node.js environment. It is recommended you upgrade to 3.2.7 or 4.3.1. (https://github.com/visionmedia/debug/issues/797)", - "dev": true, - "dependencies": { - "ms": "^2.1.1" - } - }, - "node_modules/https-proxy-agent/node_modules/ms": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", - "dev": true - }, "node_modules/human-signals": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/human-signals/-/human-signals-3.0.1.tgz", @@ -9870,18 +9813,6 @@ "node": ">=0.10.0" } }, - "node_modules/mixpanel": { - "version": "0.10.3", - "resolved": "https://registry.npmjs.org/mixpanel/-/mixpanel-0.10.3.tgz", - "integrity": "sha512-wIYr5o+1XSzJ80o3QED35K/yfPAKi5FigZXTSfcs4vltfeKbilIjNgwxdno7LrqzhjoSjmIyDWkI7D3lr7TwDw==", - "dev": true, - "dependencies": { - "https-proxy-agent": "3.0.0" - }, - "engines": { - "node": ">=6.9" - } - }, "node_modules/mkdirp": { "version": "0.5.5", "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.5.tgz", @@ -18117,15 +18048,6 @@ "integrity": "sha512-K0Ptm/47OKfQRpNQ2J/oIN/3QYiK6FwW+eJbILhsdxh2WTLdl+30o8aGdTbm5JbffpFFAg/g+zi1E+jvJha5ng==", "dev": true }, - "agent-base": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-4.3.0.tgz", - "integrity": "sha512-salcGninV0nPrwpGNn4VTXBb1SOuXQBiqbrNXoeizJsHrsL6ERFM2Ne3JUSBWRE6aeNJI2ROP/WEEIDUiDe3cg==", - "dev": true, - "requires": { - "es6-promisify": "^5.0.0" - } - }, "agentkeepalive": { "version": "4.2.1", "resolved": "https://registry.npmjs.org/agentkeepalive/-/agentkeepalive-4.2.1.tgz", @@ -20844,21 +20766,6 @@ "es6-symbol": "^3.1.1" } }, - "es6-promise": { - "version": "4.2.8", - "resolved": "https://registry.npmjs.org/es6-promise/-/es6-promise-4.2.8.tgz", - "integrity": "sha512-HJDGx5daxeIvxdBxvG2cb9g4tEvwIk3i8+nhX0yGrYmZUzbkdg8QbDevheDB8gd0//uPj4c1EQua8Q+MViT0/w==", - "dev": true - }, - "es6-promisify": { - "version": "5.0.0", - "resolved": "http://registry.npmjs.org/es6-promisify/-/es6-promisify-5.0.0.tgz", - "integrity": "sha1-UQnWLz5W6pZ8S2NQWu8IKRyKUgM=", - "dev": true, - "requires": { - "es6-promise": "^4.0.3" - } - }, "es6-symbol": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/es6-symbol/-/es6-symbol-3.1.3.tgz", @@ -22510,33 +22417,6 @@ "integrity": "sha1-7AbBDgo0wPL68Zn3/X/Hj//QPHM=", "dev": true }, - "https-proxy-agent": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-3.0.0.tgz", - "integrity": "sha512-y4jAxNEihqvBI5F3SaO2rtsjIOnnNA8sEbuiP+UhJZJHeM2NRm6c09ax2tgqme+SgUUvjao2fJXF4h3D6Cb2HQ==", - "dev": true, - "requires": { - "agent-base": "^4.3.0", - "debug": "^3.1.0" - }, - "dependencies": { - "debug": { - "version": "3.2.6", - "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.6.tgz", - "integrity": "sha512-mel+jf7nrtEl5Pn1Qx46zARXKDpBbvzezse7p7LqINmdoIk8PYP5SySaxEmYv6TZ0JyEKA1hsCId6DIhgITtWQ==", - "dev": true, - "requires": { - "ms": "^2.1.1" - } - }, - "ms": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", - "dev": true - } - } - }, "human-signals": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/human-signals/-/human-signals-3.0.1.tgz", @@ -24376,15 +24256,6 @@ } } }, - "mixpanel": { - "version": "0.10.3", - "resolved": "https://registry.npmjs.org/mixpanel/-/mixpanel-0.10.3.tgz", - "integrity": "sha512-wIYr5o+1XSzJ80o3QED35K/yfPAKi5FigZXTSfcs4vltfeKbilIjNgwxdno7LrqzhjoSjmIyDWkI7D3lr7TwDw==", - "dev": true, - "requires": { - "https-proxy-agent": "3.0.0" - } - }, "mkdirp": { "version": "0.5.5", "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.5.tgz", diff --git a/package.json b/package.json index 83637a6d..65a9f8e8 100644 --- a/package.json +++ b/package.json @@ -103,7 +103,6 @@ "livepush": "^3.5.1", "lodash": "^4.17.21", "memoizee": "^0.4.14", - "mixpanel": "^0.10.3", "mocha": "^8.3.2", "mocha-pod": "^0.6.0", "mock-fs": "^4.14.0", diff --git a/src/config/functions.ts b/src/config/functions.ts index 48cbf0f8..8bb7318e 100644 --- a/src/config/functions.ts +++ b/src/config/functions.ts @@ -2,7 +2,6 @@ import * as Bluebird from 'bluebird'; import * as _ from 'lodash'; import * as memoizee from 'memoizee'; import { promises as fs } from 'fs'; -import { URL } from 'url'; import supervisorVersion = require('../lib/supervisor-version'); @@ -114,15 +113,6 @@ export const fnSchema = { }; }); }, - mixpanelHost: () => { - return config.get('apiEndpoint').then((apiEndpoint) => { - if (!apiEndpoint) { - return null; - } - const url = new URL(apiEndpoint); - return { host: url.host, path: '/mixpanel' }; - }); - }, extendedEnvOptions: () => { return config.getMany([ 'uuid', diff --git a/src/config/schema-type.ts b/src/config/schema-type.ts index 6aa3ec52..13df3c7d 100644 --- a/src/config/schema-type.ts +++ b/src/config/schema-type.ts @@ -1,7 +1,5 @@ import * as t from 'io-ts'; -import * as constants from '../lib/constants'; - import { NullOrUndefined, PermissiveBoolean, @@ -66,10 +64,6 @@ export const schemaTypes = { type: PermissiveBoolean, default: true, }, - mixpanelToken: { - type: t.string, - default: constants.defaultMixpanelToken, - }, bootstrapRetryDelay: { type: PermissiveNumber, default: 30000, @@ -226,10 +220,6 @@ export const schemaTypes = { }), default: t.never, }, - mixpanelHost: { - type: t.union([t.null, t.interface({ host: t.string, path: t.string })]), - default: t.never, - }, extendedEnvOptions: { type: t.interface({ uuid: t.union([t.string, NullOrUndefined]), diff --git a/src/config/schema.ts b/src/config/schema.ts index ae7d4003..64bb6e10 100644 --- a/src/config/schema.ts +++ b/src/config/schema.ts @@ -54,11 +54,6 @@ export const schema = { mutable: true, removeIfNull: false, }, - mixpanelToken: { - source: 'config.json', - mutable: false, - removeIfNull: false, - }, bootstrapRetryDelay: { source: 'config.json', mutable: false, diff --git a/src/lib/constants.ts b/src/lib/constants.ts index 6e7a051b..efebe689 100644 --- a/src/lib/constants.ts +++ b/src/lib/constants.ts @@ -41,7 +41,6 @@ const constants = { configJsonPathOnHost: checkString(process.env.CONFIG_JSON_PATH), proxyvisorHookReceiver: 'http://0.0.0.0:1337', configJsonNonAtomicPath: '/boot/config.json', - defaultMixpanelToken: process.env.DEFAULT_MIXPANEL_TOKEN, supervisorNetworkInterface, allowedInterfaces: [ 'resin-vpn', diff --git a/src/supervisor.ts b/src/supervisor.ts index 30bbcdc3..2ab3384d 100644 --- a/src/supervisor.ts +++ b/src/supervisor.ts @@ -20,8 +20,6 @@ const startupConfigFields: config.ConfigKey[] = [ 'apiTimeout', 'unmanaged', 'deviceApiKey', - 'mixpanelToken', - 'mixpanelHost', 'loggingEnabled', 'localMode', 'legacyAppsPresent', From b207c019888cdae8dfb94d517a530152bc71dcbf Mon Sep 17 00:00:00 2001 From: Felipe Lalanne Date: Tue, 20 Sep 2022 11:56:22 -0300 Subject: [PATCH 3/3] Fix livepush to work with node 16 This also improves the memory efficiency of the sync mechanism by calculating the stage ids on the fly instead of storing the full build output in memory and then parsing the string. --- sync/device.ts | 55 +++++++++++++++++++++++++++++++++++++++----------- sync/init.ts | 37 +-------------------------------- sync/sync.ts | 4 ++-- 3 files changed, 46 insertions(+), 50 deletions(-) diff --git a/sync/device.ts b/sync/device.ts index c43fed1b..57216196 100644 --- a/sync/device.ts +++ b/sync/device.ts @@ -5,8 +5,9 @@ import { Builder } from 'resin-docker-build'; import { promises as fs } from 'fs'; import * as Path from 'path'; -import { Duplex, Readable, PassThrough, Stream } from 'stream'; +import { Readable } from 'stream'; import * as tar from 'tar-stream'; +import * as readline from 'readline'; import { exec } from '../src/lib/fs-utils'; @@ -52,7 +53,7 @@ export async function getDeviceArch(docker: Docker): Promise { } return arch.trim(); - } catch (e) { + } catch (e: any) { throw new Error( `Unable to get device architecture: ${e.message}.\nTry specifying the architecture with -a.`, ); @@ -68,31 +69,61 @@ export async function getCacheFrom(docker: Docker): Promise { } } +// Source: https://github.com/balena-io/balena-cli/blob/f6d668684a6f5ea8102a964ca1942b242eaa7ae2/lib/utils/device/live.ts#L539-L547 +function extractDockerArrowMessage(outputLine: string): string | undefined { + const arrowTest = /^.*\s*-+>\s*(.+)/i; + const match = arrowTest.exec(outputLine); + if (match != null) { + return match[1]; + } +} + export async function performBuild( docker: Docker, dockerfile: Dockerfile, dockerOpts: { [key: string]: any }, -): Promise { +): Promise { const builder = Builder.fromDockerode(docker); // tar the directory, but replace the dockerfile with the // livepush generated one const tarStream = await tarDirectory(Path.join(__dirname, '..'), dockerfile); - const bufStream = new PassThrough(); return new Promise((resolve, reject) => { - const chunks = [] as Buffer[]; - bufStream.on('data', (chunk) => chunks.push(Buffer.from(chunk))); + // Store the stage ids for caching + const ids = [] as string[]; builder.createBuildStream(dockerOpts, { buildSuccess: () => { - // Return the build logs - resolve(Buffer.concat(chunks).toString('utf8')); + // Return the image ids + resolve(ids); }, buildFailure: reject, - buildStream: (stream: Duplex) => { - stream.pipe(process.stdout); - stream.pipe(bufStream); - tarStream.pipe(stream); + buildStream: (input: NodeJS.ReadWriteStream) => { + // Parse the build output to get stage ids and + // for logging + let lastArrowMessage: string | undefined; + readline.createInterface({ input }).on('line', (line) => { + // If this was a FROM line, take the last found + // image id and save it as a stage id + // Source: https://github.com/balena-io/balena-cli/blob/f6d668684a6f5ea8102a964ca1942b242eaa7ae2/lib/utils/device/live.ts#L300-L325 + if ( + /step \d+(?:\/\d+)?\s*:\s*FROM/i.test(line) && + lastArrowMessage != null + ) { + ids.push(lastArrowMessage); + } else { + const msg = extractDockerArrowMessage(line); + if (msg != null) { + lastArrowMessage = msg; + } + } + + // Log the build line + console.info(line); + }); + + // stream.pipe(bufStream); + tarStream.pipe(input); }, }); }); diff --git a/sync/init.ts b/sync/init.ts index d0a267f1..a047b11a 100644 --- a/sync/init.ts +++ b/sync/init.ts @@ -14,39 +14,6 @@ interface Opts { arch?: string; } -// Source: https://github.com/balena-io/balena-cli/blob/f6d668684a6f5ea8102a964ca1942b242eaa7ae2/lib/utils/device/live.ts#L539-L547 -function extractDockerArrowMessage(outputLine: string): string | undefined { - const arrowTest = /^.*\s*-+>\s*(.+)/i; - const match = arrowTest.exec(outputLine); - if (match != null) { - return match[1]; - } -} - -// Source: https://github.com/balena-io/balena-cli/blob/f6d668684a6f5ea8102a964ca1942b242eaa7ae2/lib/utils/device/live.ts#L300-L325 -function getMultiStateImageIDs(buildLog: string): string[] { - const ids = [] as string[]; - const lines = buildLog.split(/\r?\n/); - let lastArrowMessage: string | undefined; - for (const line of lines) { - // If this was a from line, take the last found - // image id and save it - if ( - /step \d+(?:\/\d+)?\s*:\s*FROM/i.test(line) && - lastArrowMessage != null - ) { - ids.push(lastArrowMessage); - } else { - const msg = extractDockerArrowMessage(line); - if (msg != null) { - lastArrowMessage = msg; - } - } - } - - return ids; -} - function getPathPrefix(arch: string) { switch (arch) { /** @@ -74,7 +41,7 @@ export async function initDevice(opts: Opts) { const buildCache = await device.readBuildCache(opts.address); - const buildLog = await device.performBuild(opts.docker, opts.dockerfile, { + const stageImages = await device.performBuild(opts.docker, opts.dockerfile, { buildargs: { ARCH: arch, PREFIX: getPathPrefix(arch) }, t: image, labels: { 'io.balena.livepush-image': '1', 'io.balena.architecture': arch }, @@ -84,8 +51,6 @@ export async function initDevice(opts: Opts) { nocache: opts.nocache, }); - const stageImages = getMultiStateImageIDs(buildLog); - // Store the list of stage images for the next time the sync // command is called. This will only live until the device is rebooted await device.writeBuildCache(opts.address, stageImages); diff --git a/sync/sync.ts b/sync/sync.ts index 57bf21c9..e63190a8 100644 --- a/sync/sync.ts +++ b/sync/sync.ts @@ -93,12 +93,12 @@ const argv = yargs sigint = () => reject(new Error('User interrupt (Ctrl+C) received')); process.on('SIGINT', sigint); }); - } catch (e) { + } catch (e: any) { console.error('Error:', e.message); } finally { console.info('Cleaning up. Please wait ...'); await cleanup(); process.removeListener('SIGINT', sigint); - process.exit(1); + process.exit(0); } })();