Add support for auto-conversion of CRLF line endings.

Applies to commands:
 balena push
 balena build
 balena deploy --build

Change-type: minor
Resolves: #1273
Signed-off-by: Scott Lowe <scott@balena.io>

doc/cli.markdown

@@ -1802,6 +1802,10 @@ separated by a colon, e.g:
 Note that if the service name cannot be found in the composition, the entire
 left hand side of the = character will be treated as the variable name.
 
+#### --convert-eol, -l
+
+Convert line endings from CRLF (Windows format) to LF (Unix format). Source files are not modified.
+
 # Settings
 
 ## settings
@@ -1927,6 +1931,10 @@ Display full log output
 
 Path to a YAML or JSON file with passwords for a private Docker registry
 
+#### --convert-eol, -l
+
+Convert line endings from CRLF (Windows format) to LF (Unix format). Source files are not modified.
+
 #### --docker, -P <docker>
 
 Path to a local docker socket (e.g. /var/run/docker.sock)
@@ -2052,6 +2060,10 @@ Display full log output
 
 Path to a YAML or JSON file with passwords for a private Docker registry
 
+#### --convert-eol, -l
+
+Convert line endings from CRLF (Windows format) to LF (Unix format). Source files are not modified.
+
 #### --docker, -P <docker>
 
 Path to a local docker socket (e.g. /var/run/docker.sock)

lib/actions/build.coffee

@@ -41,8 +41,10 @@ buildProject = (docker, logger, composeOpts, opts) ->
 			opts.buildEmulated
 			opts.buildOpts
 			composeOpts.inlineLogs
+			opts.convertEol
 		)
 	.then ->
+		logger.outputDeferredMessages()
 		logger.logSuccess('Build succeeded!')
 	.tapCatch (e) ->
 		logger.logError('Build failed')
@@ -117,6 +119,9 @@ module.exports =
 		options.source ?= params.source
 		delete params.source
 
+		options.convertEol = options['convert-eol'] || false
+		delete options['convert-eol']
+
 		Promise.resolve(validateComposeOptions(sdk, options))
 		.then ->
 			{ application, arch, deviceType } = options
@@ -150,6 +155,7 @@ module.exports =
 						deviceType
 						buildEmulated: !!options.emulated
 						buildOpts
+						convertEol: options.convertEol
 					})
 			)
 		.asCallback(done)

lib/actions/deploy.coffee

@@ -4,6 +4,7 @@ Promise = require('bluebird')
 dockerUtils = require('../utils/docker')
 compose = require('../utils/compose')
 { registrySecretsHelp } = require('../utils/messages')
+{ ExpectedError } = require('../errors')
 
 ###
 Opts must be an object with the following keys:
@@ -60,6 +61,7 @@ deployProject = (docker, logger, composeOpts, opts) ->
 				opts.buildEmulated
 				opts.buildOpts
 				composeOpts.inlineLogs
+				opts.convertEol
 			)
 			.then (builtImages) ->
 				_.keyBy(builtImages, 'serviceName')
@@ -114,6 +116,7 @@ deployProject = (docker, logger, composeOpts, opts) ->
 					)
 			)
 	.then (release) ->
+		logger.outputDeferredMessages()
 		logger.logSuccess('Deploy succeeded!')
 		logger.logSuccess("Release: #{release.commit}")
 		console.log()
@@ -175,7 +178,7 @@ module.exports =
 			signature: 'nologupload'
 			description: "Don't upload build logs to the dashboard with image (if building)"
 			boolean: true
-		}
+		},
 	]
 	action: (params, options, done) ->
 		# compositions with many services trigger misleading warnings
@@ -196,6 +199,11 @@ module.exports =
 		appName = appName_raw || appName || options.application
 		delete options.application
 
+		options.convertEol = options['convert-eol'] || false
+		delete options['convert-eol']
+		if options.convertEol and not options.build
+			return done(new ExpectedError('The --eol-conversion flag is only valid with --build.'))
+
 		Promise.resolve(validateComposeOptions(sdk, options))
 		.then ->
 			if not appName?
@@ -213,9 +221,9 @@ module.exports =
 					return app
 			)
 			.then (app) ->
-				[ app, image, !!options.build, !options.nologupload ]
+				[ app, image, !!options.build, !options.nologupload]
 
-		.then ([ app, image, shouldPerformBuild, shouldUploadLogs ]) ->
+		.then ([ app, image, shouldPerformBuild, shouldUploadLogs, convertEol ]) ->
 			Promise.join(
 				dockerUtils.getDocker(options)
 				dockerUtils.generateBuildOpts(options)
@@ -229,6 +237,7 @@ module.exports =
 						shouldUploadLogs
 						buildEmulated: !!options.emulated
 						buildOpts
+						convertEol: options.convertEol
 					})
 			)
 		.asCallback(done)

lib/actions/push.ts

@@ -1,5 +1,5 @@
 /*
-Copyright 2016-2019 Balena Ltd.
+Copyright 2016-2020 Balena Ltd.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -113,6 +113,7 @@ export const push: CommandDefinition<
 		service?: string | string[];
 		system?: boolean;
 		env?: string | string[];
+		'convert-eol'?: boolean;
 	}
 > = {
 	signature: 'push <applicationOrDevice>',
@@ -243,6 +244,13 @@ export const push: CommandDefinition<
 				left hand side of the = character will be treated as the variable name.
 			`,
 		},
+		{
+			signature: 'convert-eol',
+			alias: 'l',
+			description: stripIndent`
+				Convert line endings from CRLF (Windows format) to LF (Unix format). Source files are not modified.`,
+			boolean: true,
+		},
 	],
 	async action(params, options, done) {
 		const sdk = (await import('balena-sdk')).fromSharedOptions();
@@ -317,6 +325,7 @@ export const push: CommandDefinition<
 							nocache: options.nocache || false,
 							registrySecrets,
 							headless: options.detached || false,
+							convertEol: options['convert-eol'] || false,
 						};
 						const args = {
 							app,
@@ -327,7 +336,6 @@ export const push: CommandDefinition<
 							sdk,
 							opts,
 						};
-
 						return await remote.startRemoteBuild(args);
 					},
 				).nodeify(done);
@@ -356,6 +364,7 @@ export const push: CommandDefinition<
 							typeof options.env === 'string'
 								? [options.env]
 								: options.env || [],
+						convertEol: options['convert-eol'] || false,
 					}),
 				)
 					.catch(BuildError, e => {

lib/utils/compose.coffee

@@ -1,6 +1,6 @@
 ###*
 # @license
-# Copyright 2017-2019 Balena Ltd.
+# Copyright 2017-2020 Balena Ltd.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -52,6 +52,12 @@ exports.appendOptions = (opts) ->
 			parameter: 'secrets.yml|.json'
 			description: 'Path to a YAML or JSON file with passwords for a private Docker registry'
 		},
+		{
+			signature: 'convert-eol'
+			description: 'Convert line endings from CRLF (Windows format) to LF (Unix format). Source files are not modified.'
+			boolean: true
+			alias: 'l'
+		}
 	]
 
 exports.generateOpts = (options) ->
@@ -131,7 +137,11 @@ exports.loadProject = (logger, projectPath, projectName, image, dockerfilePath)
 		logger.logDebug('Creating project...')
 		createProject(projectPath, composeStr, projectName)
 
-exports.tarDirectory = tarDirectory = (dir, preFinalizeCallback = null) ->
+
+exports.tarDirectory = tarDirectory = (dir, { preFinalizeCallback, convertEol } = {}) ->
+	preFinalizeCallback ?= null
+	convertEol ?= false
+
 	tar = require('tar-stream')
 	klaw = require('klaw')
 	path = require('path')
@@ -139,6 +149,7 @@ exports.tarDirectory = tarDirectory = (dir, preFinalizeCallback = null) ->
 	streamToPromise = require('stream-to-promise')
 	{ FileIgnorer } = require('./ignore')
 	{ toPosixPath } = require('resin-multibuild').PathUtils
+	{ readFileWithEolConversion } = require('./eol-conversion')
 
 	getFiles = ->
 		streamToPromise(klaw(dir))
@@ -155,7 +166,7 @@ exports.tarDirectory = tarDirectory = (dir, preFinalizeCallback = null) ->
 	.filter(ignore.filter)
 	.map (file) ->
 		relPath = path.relative(path.resolve(dir), file)
-		Promise.join relPath, fs.stat(file), fs.readFile(file),
+		Promise.join relPath, fs.stat(file), readFileWithEolConversion(file, convertEol),
 			(filename, stats, data) ->
 				pack.entry({ name: toPosixPath(filename), size: stats.size, mode: stats.mode }, data)
 	.then ->
@@ -179,7 +190,8 @@ exports.buildProject = (
 	projectPath, projectName, composition,
 	arch, deviceType,
 	emulated, buildOpts,
-	inlineLogs
+	inlineLogs,
+	convertEol
 ) ->
 	_ = require('lodash')
 	humanize = require('humanize')
@@ -214,7 +226,7 @@ exports.buildProject = (
 			return qemu.copyQemu(path.join(projectPath, d.image.context), arch)
 	.then (needsQemu) ->
 		# Tar up the directory, ready for the build stream
-		tarDirectory(projectPath)
+		tarDirectory(projectPath, { convertEol })
 		.then (tarStream) ->
 			Promise.resolve(makeBuildTasks(composition, tarStream, { arch, deviceType }, logger))
 		.map (task) ->

lib/utils/compose.d.ts

@@ -1,6 +1,6 @@
 /**
  * @license
- * Copyright 2018 Balena Ltd.
+ * Copyright 2018-2020 Balena Ltd.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -49,7 +49,12 @@ export function loadProject(
 	dockerfilePath?: string,
 ): Bluebird<ComposeProject>;
 
+interface TarDirectoryOptions {
+	preFinalizeCallback?: (pack: Pack) => void;
+	convertEol?: boolean;
+}
+
 export function tarDirectory(
 	source: string,
-	preFinalizeCallback?: (pack: Pack) => void,
+	options?: TarDirectoryOptions,
 ): Promise<Stream.Readable>;

lib/utils/device/deploy.ts

@@ -1,6 +1,6 @@
 /**
  * @license
- * Copyright 2018 Balena Ltd.
+ * Copyright 2018-2020 Balena Ltd.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -54,6 +54,7 @@ export interface DeviceDeployOptions {
 	services?: string[];
 	system: boolean;
 	env: string[];
+	convertEol: boolean;
 }
 
 interface ParsedEnvironment {
@@ -186,7 +187,9 @@ export async function deployToDevice(opts: DeviceDeployOptions): Promise<void> {
 
 	await checkBuildSecretsRequirements(docker, opts.source);
 	globalLogger.logDebug('Tarring all non-ignored files...');
-	const tarStream = await tarDirectory(opts.source);
+	const tarStream = await tarDirectory(opts.source, {
+		convertEol: opts.convertEol,
+	});
 
 	// Try to detect the device information
 	const deviceInfo = await api.getDeviceInformation();
@@ -261,6 +264,7 @@ export async function deployToDevice(opts: DeviceDeployOptions): Promise<void> {
 			);
 		}
 		globalLogger.logLivepush('Watching for file changes...');
+		globalLogger.outputDeferredMessages();
 		await Promise.all(promises);
 	} else {
 		if (opts.detached) {
@@ -272,6 +276,7 @@ export async function deployToDevice(opts: DeviceDeployOptions): Promise<void> {
 		// Now all we need to do is stream back the logs
 		const logStream = await api.getLogStream();
 		globalLogger.logInfo('Streaming device logs...');
+		globalLogger.outputDeferredMessages();
 		await displayDeviceLogs(
 			logStream,
 			globalLogger,

lib/utils/eol-conversion.ts

@@ -0,0 +1,139 @@
+/**
+ * @license
+ * Copyright 2019-2020 Balena Ltd.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import mmmagic = require('mmmagic');
+import fs = require('mz/fs');
+import Logger = require('./logger');
+
+const globalLogger = Logger.getLogger();
+
+// Define file size threshold (bytes) over which analysis/conversion is not performed.
+const LARGE_FILE_THRESHOLD = 10 * 1000 * 1000;
+
+// The list of encodings to convert is intentionally conservative for now
+const CONVERTIBLE_ENCODINGS = ['ascii', 'utf-8'];
+
+/**
+ * Attempt to detect the encoding of a data buffer
+ * @param data
+ */
+async function detectEncoding(data: Buffer): Promise<string> {
+	// Instantiate mmmagic for mime encoding analysis
+	const magic = new mmmagic.Magic(mmmagic.MAGIC_MIME_ENCODING);
+
+	// Promisify magic.detect
+	// For some reason, got 'Illegal Invocation' when using:
+	//   const detectEncoding = promisify(magic.detect);
+	return new Promise((resolve, reject) => {
+		magic.detect(data, (err, encoding) => {
+			if (err) {
+				return reject(err);
+			}
+			// mmmagic reports ascii as 'us-ascii', but node Buffer uses 'ascii'
+			encoding = encoding === 'us-ascii' ? 'ascii' : encoding;
+			return resolve(encoding);
+		});
+	});
+}
+
+/**
+ * Convert EOL (CRLF → LF) in place, i.e. modifying the input buffer.
+ * Safe for UTF-8, ASCII and 8-bit encodings (like 'latin-1', 'iso-8859-1', ...),
+ * but not safe for UTF-16 or UTF-32.
+ * Return a new buffer object sharing the same contents memory space as the
+ * input buffer (using Buffer.slice()), in order to safely reflect the new
+ * buffer size.
+ * @param buf
+ */
+function convertEolInPlace(buf: Buffer): Buffer {
+	const CR = 13;
+	const LF = 10;
+	let foundCR = false;
+	let j;
+	// Algorithm gist:
+	// - i and j are running indexes over the same buffer, but think of it as
+	//   i pointing to the input buffer, and j pointing to the output buffer.
+	// - i and j are incremented by 1 in every loop iteration, but if a LF is found
+	//   after a CR, then j is decremented by 1, and LF is written. Invariant: j <= i.
+	for (let i = (j = 0); i < buf.length; i++, j++) {
+		const b = (buf[j] = buf[i]);
+		if (b === CR) {
+			foundCR = true;
+		} else {
+			if (foundCR && b === LF) {
+				j--; // decrement index of "output buffer"
+				buf[j] = LF; // overwrite previous CR with LF
+			}
+			foundCR = false;
+		}
+	}
+	return buf.slice(0, j);
+}
+
+/**
+ * Drop-in replacement for promisified fs.readFile(<string>)
+ * Attempts to convert EOLs from CRLF to LF for supported encodings,
+ * or otherwise logs warnings.
+ * @param filepath
+ * @param convertEol When true, performs conversions, otherwise just warns.
+ */
+export async function readFileWithEolConversion(
+	filepath: string,
+	convertEol: boolean,
+): Promise<Buffer> {
+	const fileBuffer = await fs.readFile(filepath);
+
+	// Skip processing of very large files
+	const fileStats = await fs.stat(filepath);
+	if (fileStats.size > LARGE_FILE_THRESHOLD) {
+		globalLogger.logWarn(`CRLF detection skipped for large file: ${filepath}`);
+		return fileBuffer;
+	}
+
+	// Analyse encoding
+	const encoding = await detectEncoding(fileBuffer);
+
+	// Skip further processing of non-convertible encodings
+	if (!CONVERTIBLE_ENCODINGS.includes(encoding)) {
+		return fileBuffer;
+	}
+
+	// Skip further processing of files that don't contain CRLF
+	if (!fileBuffer.includes('\r\n', 0, encoding)) {
+		return fileBuffer;
+	}
+
+	if (convertEol) {
+		// Convert CRLF->LF
+		globalLogger.logInfo(
+			`Converting line endings CRLF -> LF for file: ${filepath}`,
+		);
+
+		return convertEolInPlace(fileBuffer);
+	} else {
+		// Immediate warning
+		globalLogger.logWarn(
+			`CRLF (Windows) line endings detected in file: ${filepath}`,
+		);
+		// And summary warning later
+		globalLogger.deferredLog(
+			'Windows-format line endings were detected in some files. Consider using the `--convert-eol` option.',
+			Logger.Level.WARN,
+		);
+
+		return fileBuffer;
+	}
+}

lib/utils/helpers.ts

@@ -1,5 +1,5 @@
 /*
-Copyright 2016-2019 Balena
+Copyright 2016-2020 Balena
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -354,7 +354,7 @@ function windowsCmdExeEscapeArg(arg: string): string {
 	return `"${arg.replace(/["]/g, '""')}"`;
 }
 
-/*
+/**
  * Workaround a window system bug which causes multiple rapid DNS lookups
  * to fail for mDNS.
  *

lib/utils/remote-build.ts

@@ -1,5 +1,5 @@
 /*
-Copyright 2016-2018 Balena Ltd.
+Copyright 2016-2020 Balena Ltd.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -23,10 +23,13 @@ import * as Stream from 'stream';
 import streamToPromise = require('stream-to-promise');
 import { Pack } from 'tar-stream';
 import { TypedError } from 'typed-error';
+import Logger = require('./logger');
 
 import { exitWithExpectedError } from '../utils/patterns';
 import { tarDirectory } from './compose';
 
+const globalLogger = Logger.getLogger();
+
 const DEBUG_MODE = !!process.env.DEBUG;
 
 const CURSOR_METADATA_REGEX = /([a-z]+)([0-9]+)?/;
@@ -38,6 +41,7 @@ export interface BuildOpts {
 	nocache: boolean;
 	registrySecrets: RegistrySecrets;
 	headless: boolean;
+	convertEol: boolean;
 }
 
 export interface RemoteBuild {
@@ -136,6 +140,7 @@ export async function startRemoteBuild(build: RemoteBuild): Promise<void> {
 			if (build.hadError) {
 				throw new RemoteBuildFailedError();
 			}
+			globalLogger.outputDeferredMessages();
 		});
 	}
 
@@ -289,12 +294,14 @@ async function getTarStream(build: RemoteBuild): Promise<Stream.Readable> {
 
 	try {
 		tarSpinner.start();
-		return await tarDirectory(
+		const preFinalizeCb =
-			path.resolve(build.source),
 			Object.keys(build.opts.registrySecrets).length > 0
 				? preFinalizeCallback
-				: undefined,
+				: undefined;
-		);
+		return await tarDirectory(path.resolve(build.source), {
+			preFinalizeCallback: preFinalizeCb,
+			convertEol: build.opts.convertEol,
+		});
 	} finally {
 		tarSpinner.stop();
 	}

npm-shrinkwrap.json

@@ -775,6 +775,15 @@
         "@types/node": "*"
       }
     },
+    "@types/mmmagic": {
+      "version": "0.4.16-alpha",
+      "resolved": "https://registry.npmjs.org/@types/mmmagic/-/mmmagic-0.4.16-alpha.tgz",
+      "integrity": "sha1-zM66vnBpBmPWRaMdTLzxzZ3+UIE=",
+      "dev": true,
+      "requires": {
+        "@types/node": "*"
+      }
+    },
     "@types/mocha": {
       "version": "5.2.7",
       "resolved": "https://registry.npmjs.org/@types/mocha/-/mocha-5.2.7.tgz",
@@ -8926,6 +8935,14 @@
         }
       }
     },
+    "mmmagic": {
+      "version": "0.5.3",
+      "resolved": "https://registry.npmjs.org/mmmagic/-/mmmagic-0.5.3.tgz",
+      "integrity": "sha512-xLqCu7GJYTzJczg0jafXFuh+iPzQL/ru0YYf4GiTTz8Cehru/wiXtUS8Pp8Xi77zNaiVndJ0OO1yAFci6iHyFg==",
+      "requires": {
+        "nan": "^2.13.2"
+      }
+    },
     "mocha": {
       "version": "6.2.2",
       "resolved": "https://registry.npmjs.org/mocha/-/mocha-6.2.2.tgz",

package.json

@@ -109,6 +109,7 @@
     "@types/lodash": "4.14.112",
     "@types/mixpanel": "2.14.0",
     "@types/mkdirp": "0.5.2",
+    "@types/mmmagic": "0.4.16-alpha",
     "@types/mocha": "^5.2.7",
     "@types/mz": "0.0.32",
     "@types/net-keepalive": "^0.4.0",
@@ -203,6 +204,7 @@
     "minimatch": "^3.0.4",
     "mixpanel": "^0.10.3",
     "mkdirp": "^0.5.1",
+    "mmmagic": "^0.5.3",
     "moment": "^2.24.0",
     "moment-duration-format": "^2.3.2",
     "mz": "^2.7.0",