Use webpack to join all modules

This saves around 13MB in the resulting uncompressed docker image.

Change-Type: patch
Signed-off-by: Pablo Carranza Velez <pablo@resin.io>

.gitignore

@@ -18,3 +18,4 @@ Dockerfile.runtime.*
 !Dockerfile.build.template
 !Dockerfile.runtime.template
 /build/
+/dist/

Dockerfile.build.template

@@ -21,18 +21,20 @@ RUN mkdir -p rootfs-overlay && \
 
 COPY package.json /usr/src/app/
 
-# Install only the production modules
+# Install only the production modules that have C extensions
 RUN JOBS=MAX npm install --production --no-optional --unsafe-perm \
 	&& npm dedupe
 
+COPY webpack.config.js remove-hashbang-loader.js /usr/src/app/
 COPY src /usr/src/app/src
 
 # Install devDependencies, build the coffeescript and then prune the deps
-RUN npm install --only=dev --no-optional --unsafe-perm \
+RUN cp -R node_modules node_modules_prod \
+	&& npm install --no-optional --unsafe-perm \
 	&& npm run lint \
 	&& npm run build \
-	&& npm prune --production \
-	&& npm dedupe
+	&& rm -rf node_modules \
+	&& mv node_modules_prod node_modules
 
 # Remove various uneeded filetypes in order to reduce space
 RUN find . -path '*/coverage/*' -o -path '*/test/*' -o -path '*/.nyc_output/*' \
@@ -52,7 +54,7 @@ COPY entry.sh run.sh package.json rootfs-overlay/usr/src/app/
 
 COPY inittab rootfs-overlay/etc/inittab
 
-CMD rsync -a --delete node_modules src rootfs-overlay /build
+CMD rsync -a --delete node_modules dist rootfs-overlay /build
 
 # -*- mode: dockerfile -*-
 # vi: set ft=dockerfile :

Dockerfile.runtime.template

@@ -3,7 +3,7 @@ FROM %%BASE_IMAGE_TAG%%
 
 WORKDIR /usr/src/app
 
-COPY ./build/%%ARCH%%/src ./src
+COPY ./build/%%ARCH%%/dist ./dist
 COPY ./build/%%ARCH%%/node_modules ./node_modules
 COPY ./build/%%ARCH%%/gosuper ./gosuper
 COPY ./build/%%ARCH%%/rootfs-overlay/ /

inittab

@@ -3,5 +3,5 @@
 
 stdout::sysinit:/usr/src/app/entry.sh
 
-stdout::respawn:/usr/src/app/run.sh node /usr/src/app/src/app.js
+stdout::respawn:/usr/src/app/run.sh node /usr/src/app/dist/app.js
 stdout::respawn:/usr/src/app/run.sh /usr/src/app/gosuper

package.json

@@ -9,16 +9,25 @@
   },
   "scripts": {
     "start": "./entry.sh",
-    "build": "coffee -c src",
+    "build": "webpack --optimize-minimize",
     "lint": "resin-lint src/",
     "versionist": "versionist"
   },
   "dependencies": {
+    "mkfifo": "^0.1.5",
+    "sqlite3": "^3.1.0"
+  },
+  "engines": {
+    "node": "^6.5.0"
+  },
+  "devDependencies": {
     "JSONStream": "^1.1.2",
     "blinking": "~0.0.2",
     "bluebird": "^3.5.0",
     "body-parser": "^1.12.0",
     "buffer-equal-constant-time": "^1.0.1",
+    "coffee-loader": "^0.7.3",
+    "coffee-script": "~1.11.0",
     "docker-delta": "1.0.3",
     "docker-progress": "^2.5.0",
     "docker-toolbelt": "^1.3.0",
@@ -32,24 +41,20 @@
     "memoizee": "^0.4.1",
     "mixpanel": "0.0.20",
     "network-checker": "~0.0.5",
+    "node-loader": "^0.6.0",
+    "null-loader": "^0.1.1",
     "pinejs-client": "^2.4.0",
     "pubnub": "^3.7.13",
     "request": "^2.51.0",
     "request-progress": "^2.0.1",
+    "resin-lint": "^1.3.1",
     "resin-register-device": "^3.0.0",
     "rimraf": "^2.5.4",
     "rwlock": "^5.0.0",
     "semver": "^5.3.0",
     "semver-regex": "^1.0.0",
-    "sqlite3": "^3.1.0",
-    "typed-error": "~0.1.0"
-  },
-  "engines": {
-    "node": "^6.5.0"
-  },
-  "devDependencies": {
-    "coffee-script": "~1.11.0",
-    "resin-lint": "^1.3.1",
-    "versionist": "^2.8.0"
+    "typed-error": "~0.1.0",
+    "versionist": "^2.8.0",
+    "webpack": "^3.0.0"
   }
-}
+}

remove-hashbang-loader.js

@@ -0,0 +1,6 @@
+// Some of the dependencies (e.g. JSONStream) are hybrid executable-library
+// and have a #! /usr/bin/env node at the beginning of the file.
+// This webpack loader removes it so that we have valid javascript for webpack to load.
+module.exports = function (source) {
+  return source.toString().replace(/^#! .*\n/, '')
+}

src/utils.coffee

@@ -17,7 +17,7 @@ device = require './device'
 
 exports.supervisorVersion = require('./lib/supervisor-version')
 
-configJson = require('/boot/config.json')
+configJson = JSON.parse(fs.readFileSync('/boot/config.json'))
 if Boolean(config.apiEndpoint) and !Boolean(configJson.supervisorOfflineMode)
 	mixpanelClient = mixpanel.init(config.mixpanelToken)
 else

webpack.config.js

@@ -0,0 +1,89 @@
+var webpack = require('webpack');
+var path = require('path');
+var fs = require('fs');
+var _ = require('lodash');
+var path = require('path')
+
+var externalModules = [
+	'mkfifo',
+	'sqlite3',
+	'mysql2',
+	'pg',
+	'mariasql',
+	'mssql',
+	'mysql',
+	'strong-oracle',
+	'oracle',
+	'oracledb',
+	'pg-query-stream'
+]
+
+var requiredModules = []
+var maybeOptionalModules = []
+lookForOptionalDeps = function (sourceDir) {
+	// We iterate over the node modules and mark all optional dependencies as external
+	var dirs = fs.readdirSync(sourceDir)
+	for (let dir of dirs) {
+		let packageJson = {};
+		let internalNodeModules = path.join(sourceDir, dir, 'node_modules');
+		if (fs.existsSync(internalNodeModules)) {
+			lookForOptionalDeps(internalNodeModules);
+		}
+		try {
+			packageJson = JSON.parse(fs.readFileSync(path.join(sourceDir, dir, '/package.json')));
+		}
+		catch (e) {
+			continue;
+		}
+		if (packageJson.optionalDependencies != null){
+			maybeOptionalModules = maybeOptionalModules.concat(_.keys(packageJson.optionalDependencies))
+		}
+		if (packageJson.dependencies != null){
+			requiredModules = requiredModules.concat(_.keys(packageJson.dependencies))
+		}
+	}
+}
+
+lookForOptionalDeps('./node_modules')
+externalModules.push(new RegExp('^(' + _.reject(maybeOptionalModules, requiredModules).map(_.escapeRegExp).join('|') + ')(/.*)?$'));
+
+console.log('Using the following dependencies as external:', externalModules);
+
+module.exports = {
+	entry: './src/app.coffee',
+	output: {
+		filename: 'app.js',
+		path: path.resolve(__dirname, 'dist')
+	},
+	resolve: {
+		extensions: [".js", ".json", ".coffee"]
+	},
+	target: 'node',
+	module: {
+		rules: [
+			{
+				test: /\.js$/,
+				use: require.resolve('./remove-hashbang-loader')
+			},
+			{
+				test: /\.coffee$/,
+				use: require.resolve('coffee-loader')
+			}
+		]
+	},
+	externals: (context, request, callback) => {
+		for (let m of externalModules) {
+			if ((typeof m === 'string' && m === request) || (m instanceof RegExp && m.test(request))) {
+				return callback(null, 'commonjs ' + request);
+			} else if (typeof m != 'string' && !(m instanceof RegExp)) {
+				throw new Error('Invalid entry in external modules: ' + m);
+			}
+		}
+		return callback()
+	},
+	plugins: [
+		new webpack.DefinePlugin({
+			'process.env.NODE_ENV': '"production"',
+		})
+	]
+};