From 4502f2a203444f8a3e3523b6ae48c8b921c6c7d9 Mon Sep 17 00:00:00 2001 From: Paulo Castro Date: Wed, 5 Feb 2020 22:26:55 +0000 Subject: [PATCH] Avoid loading 'mmmagic' on Linux (fix "could not load any valid magic files") Resolves: #1596 Change-type: patch --- lib/utils/eol-conversion.ts | 14 +++++++---- tests/commands/build.spec.ts | 45 ++++++++++++++++++++++++----------- tests/commands/deploy.spec.ts | 20 +++++++++++----- tests/commands/push.spec.ts | 39 ++++++++++++++++++++---------- 4 files changed, 81 insertions(+), 37 deletions(-) diff --git a/lib/utils/eol-conversion.ts b/lib/utils/eol-conversion.ts index df3540dd..4c75990c 100644 --- a/lib/utils/eol-conversion.ts +++ b/lib/utils/eol-conversion.ts @@ -14,11 +14,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import mmmagic = require('mmmagic'); -import fs = require('mz/fs'); -import Logger = require('./logger'); - -const globalLogger = Logger.getLogger(); // Define file size threshold (bytes) over which analysis/conversion is not performed. const LARGE_FILE_THRESHOLD = 10 * 1000 * 1000; @@ -31,6 +26,7 @@ const CONVERTIBLE_ENCODINGS = ['ascii', 'utf-8']; * @param data */ async function detectEncoding(data: Buffer): Promise { + const mmmagic = await import('mmmagic'); // Instantiate mmmagic for mime encoding analysis const magic = new mmmagic.Magic(mmmagic.MAGIC_MIME_ENCODING); @@ -94,8 +90,16 @@ export async function readFileWithEolConversion( filepath: string, convertEol: boolean, ): Promise { + const { fs } = await import('mz'); const fileBuffer = await fs.readFile(filepath); + if (process.platform !== 'win32') { + return fileBuffer; + } + + const Logger = await import('./logger'); + const globalLogger = Logger.getLogger(); + // Skip processing of very large files const fileStats = await fs.stat(filepath); if (fileStats.size > LARGE_FILE_THRESHOLD) { diff --git a/tests/commands/build.spec.ts b/tests/commands/build.spec.ts index 5af99203..7c1658a4 100644 --- a/tests/commands/build.spec.ts +++ b/tests/commands/build.spec.ts @@ -105,28 +105,37 @@ describe('balena build', function() { `build ${projectPath} --deviceType nuc --arch amd64`, ); + const extraLines = [ + `[Info] Creating default composition with source: ${projectPath}`, + ]; + if (process.platform === 'win32') { + extraLines.push( + `[Warn] CRLF (Windows) line endings detected in file: ${path.join( + projectPath, + 'src', + 'windows-crlf.sh', + )}`, + ); + } + expect(err).to.have.members([]); expect( cleanOutput(out).map(line => line.replace(/\s{2,}/g, ' ')), ).to.include.members([ - `[Info] Creating default composition with source: ${projectPath}`, ...expectedResponses[responseFilename], - `[Warn] CRLF (Windows) line endings detected in file: ${path.join( - projectPath, - 'src', - 'windows-crlf.sh', - )}`, + ...extraLines, ]); }); it('should create the expected tar stream (single container, --convert-eol)', async () => { + const windows = process.platform === 'win32'; const projectPath = path.join(projectsPath, 'no-docker-compose', 'basic'); const expectedFiles: TarStreamFiles = { 'src/start.sh': { fileSize: 89, type: 'file' }, 'src/windows-crlf.sh': { - fileSize: 68, + fileSize: windows ? 68 : 70, type: 'file', - testStream: expectStreamNoCRLF, + testStream: windows ? expectStreamNoCRLF : undefined, }, Dockerfile: { fileSize: 88, type: 'file' }, 'Dockerfile-alt': { fileSize: 30, type: 'file' }, @@ -154,17 +163,25 @@ describe('balena build', function() { `build ${projectPath} --deviceType nuc --arch amd64 --convert-eol`, ); + const extraLines = [ + `[Info] Creating default composition with source: ${projectPath}`, + ]; + if (windows) { + extraLines.push( + `[Info] Converting line endings CRLF -> LF for file: ${path.join( + projectPath, + 'src', + 'windows-crlf.sh', + )}`, + ); + } + expect(err).to.have.members([]); expect( cleanOutput(out).map(line => line.replace(/\s{2,}/g, ' ')), ).to.include.members([ - `[Info] Creating default composition with source: ${projectPath}`, - `[Info] Converting line endings CRLF -> LF for file: ${path.join( - projectPath, - 'src', - 'windows-crlf.sh', - )}`, ...expectedResponses[responseFilename], + ...extraLines, ]); }); }); diff --git a/tests/commands/deploy.spec.ts b/tests/commands/deploy.spec.ts index ed5e58c8..b03b4ae3 100644 --- a/tests/commands/deploy.spec.ts +++ b/tests/commands/deploy.spec.ts @@ -124,17 +124,25 @@ describe('balena deploy', function() { `deploy testApp --build --source ${projectPath}`, ); + const extraLines = [ + `[Info] Creating default composition with source: ${projectPath}`, + ]; + if (process.platform === 'win32') { + extraLines.push( + `[Warn] CRLF (Windows) line endings detected in file: ${path.join( + projectPath, + 'src', + 'windows-crlf.sh', + )}`, + ); + } + expect(err).to.have.members([]); expect( cleanOutput(out).map(line => line.replace(/\s{2,}/g, ' ')), ).to.include.members([ - `[Info] Creating default composition with source: ${projectPath}`, ...expectedResponses[responseFilename], - `[Warn] CRLF (Windows) line endings detected in file: ${path.join( - projectPath, - 'src', - 'windows-crlf.sh', - )}`, + ...extraLines, ]); }); }); diff --git a/tests/commands/push.spec.ts b/tests/commands/push.spec.ts index eb39cd96..55f5ac72 100644 --- a/tests/commands/push.spec.ts +++ b/tests/commands/push.spec.ts @@ -133,14 +133,21 @@ describe('balena push', function() { `push testApp --source ${projectPath}`, ); + const extraLines = []; + if (process.platform === 'win32') { + extraLines.push( + `[Warn] CRLF (Windows) line endings detected in file: ${path.join( + projectPath, + 'src', + 'windows-crlf.sh', + )}`, + ); + } + expect(err).to.have.members([]); expect(tweakOutput(out)).to.include.members([ ...expectedResponses[responseFilename], - `[Warn] CRLF (Windows) line endings detected in file: ${path.join( - projectPath, - 'src', - 'windows-crlf.sh', - )}`, + ...extraLines, ]); }); @@ -187,13 +194,14 @@ describe('balena push', function() { }); it('should create the expected tar stream (single container, --convert-eol)', async () => { + const windows = process.platform === 'win32'; const projectPath = path.join(projectsPath, 'no-docker-compose', 'basic'); const expectedFiles: TarStreamFiles = { 'src/start.sh': { fileSize: 89, type: 'file' }, 'src/windows-crlf.sh': { - fileSize: 68, + fileSize: windows ? 68 : 70, type: 'file', - testStream: expectStreamNoCRLF, + testStream: windows ? expectStreamNoCRLF : undefined, }, Dockerfile: { fileSize: 88, type: 'file' }, 'Dockerfile-alt': { fileSize: 30, type: 'file' }, @@ -220,14 +228,21 @@ describe('balena push', function() { `push testApp --source ${projectPath} --convert-eol`, ); + const extraLines = []; + if (windows) { + extraLines.push( + `[Info] Converting line endings CRLF -> LF for file: ${path.join( + projectPath, + 'src', + 'windows-crlf.sh', + )}`, + ); + } + expect(err).to.have.members([]); expect(tweakOutput(out)).to.include.members([ ...expectedResponses[responseFilename], - `[Info] Converting line endings CRLF -> LF for file: ${path.join( - projectPath, - 'src', - 'windows-crlf.sh', - )}`, + ...extraLines, ]); }); });