2020-01-29 11:10:59 +00:00
|
|
|
/**
|
|
|
|
* @license
|
|
|
|
* Copyright 2019-2020 Balena Ltd.
|
|
|
|
*
|
|
|
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
|
* you may not use this file except in compliance with the License.
|
|
|
|
* You may obtain a copy of the License at
|
|
|
|
*
|
|
|
|
* http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
*
|
|
|
|
* Unless required by applicable law or agreed to in writing, software
|
|
|
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
* See the License for the specific language governing permissions and
|
|
|
|
* limitations under the License.
|
|
|
|
*/
|
|
|
|
|
|
|
|
// Define file size threshold (bytes) over which analysis/conversion is not performed.
|
|
|
|
const LARGE_FILE_THRESHOLD = 10 * 1000 * 1000;
|
|
|
|
|
|
|
|
// The list of encodings to convert is intentionally conservative for now
|
|
|
|
const CONVERTIBLE_ENCODINGS = ['ascii', 'utf-8'];
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Attempt to detect the encoding of a data buffer
|
|
|
|
* @param data
|
|
|
|
*/
|
|
|
|
async function detectEncoding(data: Buffer): Promise<string> {
|
2020-02-05 22:26:55 +00:00
|
|
|
const mmmagic = await import('mmmagic');
|
2020-01-29 11:10:59 +00:00
|
|
|
// Instantiate mmmagic for mime encoding analysis
|
|
|
|
const magic = new mmmagic.Magic(mmmagic.MAGIC_MIME_ENCODING);
|
|
|
|
|
|
|
|
// Promisify magic.detect
|
|
|
|
// For some reason, got 'Illegal Invocation' when using:
|
|
|
|
// const detectEncoding = promisify(magic.detect);
|
|
|
|
return new Promise((resolve, reject) => {
|
|
|
|
magic.detect(data, (err, encoding) => {
|
|
|
|
if (err) {
|
|
|
|
return reject(err);
|
|
|
|
}
|
|
|
|
// mmmagic reports ascii as 'us-ascii', but node Buffer uses 'ascii'
|
|
|
|
encoding = encoding === 'us-ascii' ? 'ascii' : encoding;
|
|
|
|
return resolve(encoding);
|
|
|
|
});
|
|
|
|
});
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Convert EOL (CRLF → LF) in place, i.e. modifying the input buffer.
|
|
|
|
* Safe for UTF-8, ASCII and 8-bit encodings (like 'latin-1', 'iso-8859-1', ...),
|
|
|
|
* but not safe for UTF-16 or UTF-32.
|
|
|
|
* Return a new buffer object sharing the same contents memory space as the
|
|
|
|
* input buffer (using Buffer.slice()), in order to safely reflect the new
|
|
|
|
* buffer size.
|
|
|
|
* @param buf
|
|
|
|
*/
|
2020-02-01 00:29:51 +00:00
|
|
|
export function convertEolInPlace(buf: Buffer): Buffer {
|
2020-01-29 11:10:59 +00:00
|
|
|
const CR = 13;
|
|
|
|
const LF = 10;
|
|
|
|
let foundCR = false;
|
|
|
|
let j;
|
|
|
|
// Algorithm gist:
|
|
|
|
// - i and j are running indexes over the same buffer, but think of it as
|
|
|
|
// i pointing to the input buffer, and j pointing to the output buffer.
|
|
|
|
// - i and j are incremented by 1 in every loop iteration, but if a LF is found
|
|
|
|
// after a CR, then j is decremented by 1, and LF is written. Invariant: j <= i.
|
|
|
|
for (let i = (j = 0); i < buf.length; i++, j++) {
|
|
|
|
const b = (buf[j] = buf[i]);
|
|
|
|
if (b === CR) {
|
|
|
|
foundCR = true;
|
|
|
|
} else {
|
|
|
|
if (foundCR && b === LF) {
|
|
|
|
j--; // decrement index of "output buffer"
|
|
|
|
buf[j] = LF; // overwrite previous CR with LF
|
|
|
|
}
|
|
|
|
foundCR = false;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return buf.slice(0, j);
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Drop-in replacement for promisified fs.readFile(<string>)
|
|
|
|
* Attempts to convert EOLs from CRLF to LF for supported encodings,
|
|
|
|
* or otherwise logs warnings.
|
|
|
|
* @param filepath
|
|
|
|
* @param convertEol When true, performs conversions, otherwise just warns.
|
|
|
|
*/
|
|
|
|
export async function readFileWithEolConversion(
|
|
|
|
filepath: string,
|
|
|
|
convertEol: boolean,
|
|
|
|
): Promise<Buffer> {
|
2020-02-05 22:26:55 +00:00
|
|
|
const { fs } = await import('mz');
|
2020-01-29 11:10:59 +00:00
|
|
|
const fileBuffer = await fs.readFile(filepath);
|
2020-02-05 22:26:55 +00:00
|
|
|
const Logger = await import('./logger');
|
|
|
|
const globalLogger = Logger.getLogger();
|
|
|
|
|
2020-01-29 11:10:59 +00:00
|
|
|
// Skip processing of very large files
|
|
|
|
const fileStats = await fs.stat(filepath);
|
|
|
|
if (fileStats.size > LARGE_FILE_THRESHOLD) {
|
|
|
|
globalLogger.logWarn(`CRLF detection skipped for large file: ${filepath}`);
|
|
|
|
return fileBuffer;
|
|
|
|
}
|
|
|
|
|
|
|
|
// Analyse encoding
|
|
|
|
const encoding = await detectEncoding(fileBuffer);
|
|
|
|
|
|
|
|
// Skip further processing of non-convertible encodings
|
|
|
|
if (!CONVERTIBLE_ENCODINGS.includes(encoding)) {
|
|
|
|
return fileBuffer;
|
|
|
|
}
|
|
|
|
|
|
|
|
// Skip further processing of files that don't contain CRLF
|
|
|
|
if (!fileBuffer.includes('\r\n', 0, encoding)) {
|
|
|
|
return fileBuffer;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (convertEol) {
|
|
|
|
// Convert CRLF->LF
|
|
|
|
globalLogger.logInfo(
|
|
|
|
`Converting line endings CRLF -> LF for file: ${filepath}`,
|
|
|
|
);
|
|
|
|
|
|
|
|
return convertEolInPlace(fileBuffer);
|
|
|
|
} else {
|
|
|
|
// Immediate warning
|
|
|
|
globalLogger.logWarn(
|
|
|
|
`CRLF (Windows) line endings detected in file: ${filepath}`,
|
|
|
|
);
|
|
|
|
// And summary warning later
|
|
|
|
globalLogger.deferredLog(
|
|
|
|
'Windows-format line endings were detected in some files. Consider using the `--convert-eol` option.',
|
|
|
|
Logger.Level.WARN,
|
|
|
|
);
|
|
|
|
|
|
|
|
return fileBuffer;
|
|
|
|
}
|
|
|
|
}
|