From 17909390461e1e131f67c2d12003445ddbff94f3 Mon Sep 17 00:00:00 2001 From: Pablo Carranza Velez Date: Fri, 30 Jun 2017 21:07:02 -0700 Subject: [PATCH] Use webpack to join all modules This saves around 13MB in the resulting uncompressed docker image. Change-Type: patch Signed-off-by: Pablo Carranza Velez --- .gitignore | 1 + Dockerfile.build.template | 12 ++--- Dockerfile.runtime.template | 2 +- inittab | 2 +- package.json | 29 +++++++----- remove-hashbang-loader.js | 6 +++ src/utils.coffee | 2 +- webpack.config.js | 89 +++++++++++++++++++++++++++++++++++++ 8 files changed, 123 insertions(+), 20 deletions(-) create mode 100644 remove-hashbang-loader.js create mode 100644 webpack.config.js diff --git a/.gitignore b/.gitignore index 52754ef1..c71493f1 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,4 @@ Dockerfile.runtime.* !Dockerfile.build.template !Dockerfile.runtime.template /build/ +/dist/ diff --git a/Dockerfile.build.template b/Dockerfile.build.template index 74d41295..6b3fb8fc 100644 --- a/Dockerfile.build.template +++ b/Dockerfile.build.template @@ -21,18 +21,20 @@ RUN mkdir -p rootfs-overlay && \ COPY package.json /usr/src/app/ -# Install only the production modules +# Install only the production modules that have C extensions RUN JOBS=MAX npm install --production --no-optional --unsafe-perm \ && npm dedupe +COPY webpack.config.js remove-hashbang-loader.js /usr/src/app/ COPY src /usr/src/app/src # Install devDependencies, build the coffeescript and then prune the deps -RUN npm install --only=dev --no-optional --unsafe-perm \ +RUN cp -R node_modules node_modules_prod \ + && npm install --no-optional --unsafe-perm \ && npm run lint \ && npm run build \ - && npm prune --production \ - && npm dedupe + && rm -rf node_modules \ + && mv node_modules_prod node_modules # Remove various uneeded filetypes in order to reduce space RUN find . -path '*/coverage/*' -o -path '*/test/*' -o -path '*/.nyc_output/*' \ @@ -52,7 +54,7 @@ COPY entry.sh run.sh package.json rootfs-overlay/usr/src/app/ COPY inittab rootfs-overlay/etc/inittab -CMD rsync -a --delete node_modules src rootfs-overlay /build +CMD rsync -a --delete node_modules dist rootfs-overlay /build # -*- mode: dockerfile -*- # vi: set ft=dockerfile : diff --git a/Dockerfile.runtime.template b/Dockerfile.runtime.template index db25b086..2d8b39f4 100644 --- a/Dockerfile.runtime.template +++ b/Dockerfile.runtime.template @@ -3,7 +3,7 @@ FROM %%BASE_IMAGE_TAG%% WORKDIR /usr/src/app -COPY ./build/%%ARCH%%/src ./src +COPY ./build/%%ARCH%%/dist ./dist COPY ./build/%%ARCH%%/node_modules ./node_modules COPY ./build/%%ARCH%%/gosuper ./gosuper COPY ./build/%%ARCH%%/rootfs-overlay/ / diff --git a/inittab b/inittab index 2324807a..4b9a0a1d 100644 --- a/inittab +++ b/inittab @@ -3,5 +3,5 @@ stdout::sysinit:/usr/src/app/entry.sh -stdout::respawn:/usr/src/app/run.sh node /usr/src/app/src/app.js +stdout::respawn:/usr/src/app/run.sh node /usr/src/app/dist/app.js stdout::respawn:/usr/src/app/run.sh /usr/src/app/gosuper diff --git a/package.json b/package.json index 36537b74..4684eb88 100644 --- a/package.json +++ b/package.json @@ -9,16 +9,25 @@ }, "scripts": { "start": "./entry.sh", - "build": "coffee -c src", + "build": "webpack --optimize-minimize", "lint": "resin-lint src/", "versionist": "versionist" }, "dependencies": { + "mkfifo": "^0.1.5", + "sqlite3": "^3.1.0" + }, + "engines": { + "node": "^6.5.0" + }, + "devDependencies": { "JSONStream": "^1.1.2", "blinking": "~0.0.2", "bluebird": "^3.5.0", "body-parser": "^1.12.0", "buffer-equal-constant-time": "^1.0.1", + "coffee-loader": "^0.7.3", + "coffee-script": "~1.11.0", "docker-delta": "1.0.3", "docker-progress": "^2.5.0", "docker-toolbelt": "^1.3.0", @@ -32,24 +41,20 @@ "memoizee": "^0.4.1", "mixpanel": "0.0.20", "network-checker": "~0.0.5", + "node-loader": "^0.6.0", + "null-loader": "^0.1.1", "pinejs-client": "^2.4.0", "pubnub": "^3.7.13", "request": "^2.51.0", "request-progress": "^2.0.1", + "resin-lint": "^1.3.1", "resin-register-device": "^3.0.0", "rimraf": "^2.5.4", "rwlock": "^5.0.0", "semver": "^5.3.0", "semver-regex": "^1.0.0", - "sqlite3": "^3.1.0", - "typed-error": "~0.1.0" - }, - "engines": { - "node": "^6.5.0" - }, - "devDependencies": { - "coffee-script": "~1.11.0", - "resin-lint": "^1.3.1", - "versionist": "^2.8.0" + "typed-error": "~0.1.0", + "versionist": "^2.8.0", + "webpack": "^3.0.0" } -} \ No newline at end of file +} diff --git a/remove-hashbang-loader.js b/remove-hashbang-loader.js new file mode 100644 index 00000000..53f25bdd --- /dev/null +++ b/remove-hashbang-loader.js @@ -0,0 +1,6 @@ +// Some of the dependencies (e.g. JSONStream) are hybrid executable-library +// and have a #! /usr/bin/env node at the beginning of the file. +// This webpack loader removes it so that we have valid javascript for webpack to load. +module.exports = function (source) { + return source.toString().replace(/^#! .*\n/, '') +} diff --git a/src/utils.coffee b/src/utils.coffee index 390134f2..77116c5b 100644 --- a/src/utils.coffee +++ b/src/utils.coffee @@ -17,7 +17,7 @@ device = require './device' exports.supervisorVersion = require('./lib/supervisor-version') -configJson = require('/boot/config.json') +configJson = JSON.parse(fs.readFileSync('/boot/config.json')) if Boolean(config.apiEndpoint) and !Boolean(configJson.supervisorOfflineMode) mixpanelClient = mixpanel.init(config.mixpanelToken) else diff --git a/webpack.config.js b/webpack.config.js new file mode 100644 index 00000000..24c57b99 --- /dev/null +++ b/webpack.config.js @@ -0,0 +1,89 @@ +var webpack = require('webpack'); +var path = require('path'); +var fs = require('fs'); +var _ = require('lodash'); +var path = require('path') + +var externalModules = [ + 'mkfifo', + 'sqlite3', + 'mysql2', + 'pg', + 'mariasql', + 'mssql', + 'mysql', + 'strong-oracle', + 'oracle', + 'oracledb', + 'pg-query-stream' +] + +var requiredModules = [] +var maybeOptionalModules = [] +lookForOptionalDeps = function (sourceDir) { + // We iterate over the node modules and mark all optional dependencies as external + var dirs = fs.readdirSync(sourceDir) + for (let dir of dirs) { + let packageJson = {}; + let internalNodeModules = path.join(sourceDir, dir, 'node_modules'); + if (fs.existsSync(internalNodeModules)) { + lookForOptionalDeps(internalNodeModules); + } + try { + packageJson = JSON.parse(fs.readFileSync(path.join(sourceDir, dir, '/package.json'))); + } + catch (e) { + continue; + } + if (packageJson.optionalDependencies != null){ + maybeOptionalModules = maybeOptionalModules.concat(_.keys(packageJson.optionalDependencies)) + } + if (packageJson.dependencies != null){ + requiredModules = requiredModules.concat(_.keys(packageJson.dependencies)) + } + } +} + +lookForOptionalDeps('./node_modules') +externalModules.push(new RegExp('^(' + _.reject(maybeOptionalModules, requiredModules).map(_.escapeRegExp).join('|') + ')(/.*)?$')); + +console.log('Using the following dependencies as external:', externalModules); + +module.exports = { + entry: './src/app.coffee', + output: { + filename: 'app.js', + path: path.resolve(__dirname, 'dist') + }, + resolve: { + extensions: [".js", ".json", ".coffee"] + }, + target: 'node', + module: { + rules: [ + { + test: /\.js$/, + use: require.resolve('./remove-hashbang-loader') + }, + { + test: /\.coffee$/, + use: require.resolve('coffee-loader') + } + ] + }, + externals: (context, request, callback) => { + for (let m of externalModules) { + if ((typeof m === 'string' && m === request) || (m instanceof RegExp && m.test(request))) { + return callback(null, 'commonjs ' + request); + } else if (typeof m != 'string' && !(m instanceof RegExp)) { + throw new Error('Invalid entry in external modules: ' + m); + } + } + return callback() + }, + plugins: [ + new webpack.DefinePlugin({ + 'process.env.NODE_ENV': '"production"', + }) + ] +};