From f9743b269ac3e54562a5074cd32835d66561eef6 Mon Sep 17 00:00:00 2001 From: Paulo Castro Date: Sat, 1 Feb 2020 00:29:51 +0000 Subject: [PATCH] Add more tests for push/build/deploy commands (--convert-eol) Change-type: patch Signed-off-by: Paulo Castro --- lib/utils/eol-conversion.ts | 2 +- tests/commands/build.spec.ts | 60 +++++++++++++++++- tests/commands/deploy.spec.ts | 10 ++- tests/commands/push.spec.ts | 63 +++++++++++++++++-- tests/helpers.ts | 46 +++++++++++--- .../no-docker-compose/basic/Dockerfile | 6 +- .../basic/src/windows-crlf.sh | 2 + tests/utils/eol-conversion.spec.ts | 55 ++++++++++++++++ 8 files changed, 223 insertions(+), 21 deletions(-) create mode 100644 tests/test-data/projects/no-docker-compose/basic/src/windows-crlf.sh create mode 100644 tests/utils/eol-conversion.spec.ts diff --git a/lib/utils/eol-conversion.ts b/lib/utils/eol-conversion.ts index e7f0e7f8..df3540dd 100644 --- a/lib/utils/eol-conversion.ts +++ b/lib/utils/eol-conversion.ts @@ -58,7 +58,7 @@ async function detectEncoding(data: Buffer): Promise { * buffer size. * @param buf */ -function convertEolInPlace(buf: Buffer): Buffer { +export function convertEolInPlace(buf: Buffer): Buffer { const CR = 13; const LF = 10; let foundCR = false; diff --git a/tests/commands/build.spec.ts b/tests/commands/build.spec.ts index fba51c9a..5af99203 100644 --- a/tests/commands/build.spec.ts +++ b/tests/commands/build.spec.ts @@ -27,6 +27,7 @@ import { BalenaAPIMock } from '../balena-api-mock'; import { DockerMock, dockerResponsePath } from '../docker-mock'; import { cleanOutput, + expectStreamNoCRLF, inspectTarStream, runCommand, TarStreamFiles, @@ -73,11 +74,12 @@ describe('balena build', function() { docker.done(); }); - it('should create the expected tar stream', async () => { + it('should create the expected tar stream (single container)', async () => { const projectPath = path.join(projectsPath, 'no-docker-compose', 'basic'); const expectedFiles: TarStreamFiles = { 'src/start.sh': { fileSize: 89, type: 'file' }, - Dockerfile: { fileSize: 85, type: 'file' }, + 'src/windows-crlf.sh': { fileSize: 70, type: 'file' }, + Dockerfile: { fileSize: 88, type: 'file' }, 'Dockerfile-alt': { fileSize: 30, type: 'file' }, }; const responseFilename = 'build-POST.json'; @@ -109,6 +111,60 @@ describe('balena build', function() { ).to.include.members([ `[Info] Creating default composition with source: ${projectPath}`, ...expectedResponses[responseFilename], + `[Warn] CRLF (Windows) line endings detected in file: ${path.join( + projectPath, + 'src', + 'windows-crlf.sh', + )}`, + ]); + }); + + it('should create the expected tar stream (single container, --convert-eol)', async () => { + const projectPath = path.join(projectsPath, 'no-docker-compose', 'basic'); + const expectedFiles: TarStreamFiles = { + 'src/start.sh': { fileSize: 89, type: 'file' }, + 'src/windows-crlf.sh': { + fileSize: 68, + type: 'file', + testStream: expectStreamNoCRLF, + }, + Dockerfile: { fileSize: 88, type: 'file' }, + 'Dockerfile-alt': { fileSize: 30, type: 'file' }, + }; + const responseFilename = 'build-POST.json'; + const responseBody = await fs.readFile( + path.join(dockerResponsePath, responseFilename), + 'utf8', + ); + + docker.expectPostBuild({ + tag: 'basic_main', + responseCode: 200, + responseBody, + checkURI: async (uri: string) => { + const url = new URL(uri, 'http://test.net/'); + const queryParams = Array.from(url.searchParams.entries()); + expect(queryParams).to.have.deep.members(commonQueryParams); + }, + checkBuildRequestBody: (buildRequestBody: string) => + inspectTarStream(buildRequestBody, expectedFiles, projectPath, expect), + }); + + const { out, err } = await runCommand( + `build ${projectPath} --deviceType nuc --arch amd64 --convert-eol`, + ); + + expect(err).to.have.members([]); + expect( + cleanOutput(out).map(line => line.replace(/\s{2,}/g, ' ')), + ).to.include.members([ + `[Info] Creating default composition with source: ${projectPath}`, + `[Info] Converting line endings CRLF -> LF for file: ${path.join( + projectPath, + 'src', + 'windows-crlf.sh', + )}`, + ...expectedResponses[responseFilename], ]); }); }); diff --git a/tests/commands/deploy.spec.ts b/tests/commands/deploy.spec.ts index 2f5b8fa2..ed5e58c8 100644 --- a/tests/commands/deploy.spec.ts +++ b/tests/commands/deploy.spec.ts @@ -93,11 +93,12 @@ describe('balena deploy', function() { docker.done(); }); - it('should create the expected --build tar stream', async () => { + it('should create the expected --build tar stream (single container)', async () => { const projectPath = path.join(projectsPath, 'no-docker-compose', 'basic'); const expectedFiles: TarStreamFiles = { 'src/start.sh': { fileSize: 89, type: 'file' }, - Dockerfile: { fileSize: 85, type: 'file' }, + 'src/windows-crlf.sh': { fileSize: 70, type: 'file' }, + Dockerfile: { fileSize: 88, type: 'file' }, 'Dockerfile-alt': { fileSize: 30, type: 'file' }, }; const responseFilename = 'build-POST.json'; @@ -129,6 +130,11 @@ describe('balena deploy', function() { ).to.include.members([ `[Info] Creating default composition with source: ${projectPath}`, ...expectedResponses[responseFilename], + `[Warn] CRLF (Windows) line endings detected in file: ${path.join( + projectPath, + 'src', + 'windows-crlf.sh', + )}`, ]); }); }); diff --git a/tests/commands/push.spec.ts b/tests/commands/push.spec.ts index 90417883..eb39cd96 100644 --- a/tests/commands/push.spec.ts +++ b/tests/commands/push.spec.ts @@ -27,6 +27,7 @@ import { BalenaAPIMock } from '../balena-api-mock'; import { BuilderMock, builderResponsePath } from '../builder-mock'; import { cleanOutput, + expectStreamNoCRLF, inspectTarStream, runCommand, TarStreamFiles, @@ -106,7 +107,8 @@ describe('balena push', function() { const projectPath = path.join(projectsPath, 'no-docker-compose', 'basic'); const expectedFiles: TarStreamFiles = { 'src/start.sh': { fileSize: 89, type: 'file' }, - Dockerfile: { fileSize: 85, type: 'file' }, + 'src/windows-crlf.sh': { fileSize: 70, type: 'file' }, + Dockerfile: { fileSize: 88, type: 'file' }, 'Dockerfile-alt': { fileSize: 30, type: 'file' }, }; const responseFilename = 'build-POST-v3.json'; @@ -132,16 +134,22 @@ describe('balena push', function() { ); expect(err).to.have.members([]); - expect(tweakOutput(out)).to.include.members( - expectedResponses[responseFilename], - ); + expect(tweakOutput(out)).to.include.members([ + ...expectedResponses[responseFilename], + `[Warn] CRLF (Windows) line endings detected in file: ${path.join( + projectPath, + 'src', + 'windows-crlf.sh', + )}`, + ]); }); it('should create the expected tar stream (alternative Dockerfile)', async () => { const projectPath = path.join(projectsPath, 'no-docker-compose', 'basic'); const expectedFiles: TarStreamFiles = { 'src/start.sh': { fileSize: 89, type: 'file' }, - Dockerfile: { fileSize: 85, type: 'file' }, + 'src/windows-crlf.sh': { fileSize: 70, type: 'file' }, + Dockerfile: { fileSize: 88, type: 'file' }, 'Dockerfile-alt': { fileSize: 30, type: 'file' }, }; const responseFilename = 'build-POST-v3.json'; @@ -177,4 +185,49 @@ describe('balena push', function() { expectedResponses[responseFilename], ); }); + + it('should create the expected tar stream (single container, --convert-eol)', async () => { + const projectPath = path.join(projectsPath, 'no-docker-compose', 'basic'); + const expectedFiles: TarStreamFiles = { + 'src/start.sh': { fileSize: 89, type: 'file' }, + 'src/windows-crlf.sh': { + fileSize: 68, + type: 'file', + testStream: expectStreamNoCRLF, + }, + Dockerfile: { fileSize: 88, type: 'file' }, + 'Dockerfile-alt': { fileSize: 30, type: 'file' }, + }; + const responseFilename = 'build-POST-v3.json'; + const responseBody = await fs.readFile( + path.join(builderResponsePath, responseFilename), + 'utf8', + ); + + builder.expectPostBuild({ + responseCode: 200, + responseBody, + checkURI: async (uri: string) => { + const url = new URL(uri, 'http://test.net/'); + const queryParams = Array.from(url.searchParams.entries()); + expect(queryParams).to.have.deep.members(commonQueryParams); + }, + checkBuildRequestBody: (buildRequestBody: string | Buffer) => + inspectTarStream(buildRequestBody, expectedFiles, projectPath, expect), + }); + + const { out, err } = await runCommand( + `push testApp --source ${projectPath} --convert-eol`, + ); + + expect(err).to.have.members([]); + expect(tweakOutput(out)).to.include.members([ + ...expectedResponses[responseFilename], + `[Info] Converting line endings CRLF -> LF for file: ${path.join( + projectPath, + 'src', + 'windows-crlf.sh', + )}`, + ]); + }); }); diff --git a/tests/helpers.ts b/tests/helpers.ts index 8ddcdcbd..c96101e0 100644 --- a/tests/helpers.ts +++ b/tests/helpers.ts @@ -18,6 +18,7 @@ // tslint:disable-next-line:no-var-requires require('./config-tests'); // required for side effects +import { stripIndent } from 'common-tags'; import intercept = require('intercept-stdout'); import * as _ from 'lodash'; import { fs } from 'mz'; @@ -117,6 +118,7 @@ export interface TarStreamFiles { [filePath: string]: { fileSize: number; type: tar.Headers['type']; + testStream?: (header: tar.Headers, stream: Readable) => Promise; }; } @@ -159,13 +161,12 @@ export async function inspectTarStream( fileSize: header.size || 0, type: header.type, }; - const [buf, buf2] = await Promise.all([ - streamToBuffer(stream), - fs.readFile( - path.join(projectPath, PathUtils.toNativePath(header.name)), - ), - ]); - expect(buf.equals(buf2)).to.be.true; + const expected = expectedFiles[header.name]; + if (expected && expected.testStream) { + await expected.testStream(header, stream); + } else { + await defaultTestStream(header, stream, projectPath, expect); + } } } catch (err) { reject(err); @@ -180,5 +181,34 @@ export async function inspectTarStream( sourceTarStream.pipe(extract); }); - expect(found).to.deep.equal(expectedFiles); + expect(found).to.deep.equal( + _.mapValues(expectedFiles, v => _.omit(v, 'testStream')), + ); +} + +/** Check that a tar stream entry matches the project contents in the filesystem */ +async function defaultTestStream( + header: tar.Headers, + stream: Readable, + projectPath: string, + expect: Chai.ExpectStatic, +): Promise { + const [buf, buf2] = await Promise.all([ + streamToBuffer(stream), + fs.readFile(path.join(projectPath, PathUtils.toNativePath(header.name))), + ]); + const msg = stripIndent` + contents mismatch for tar stream entry "${header.name}" + stream length=${buf.length}, filesystem length=${buf2.length}`; + expect(buf.equals(buf2), msg).to.be.true; +} + +/** Test a tar stream entry for the absence of Windows CRLF line breaks */ +export async function expectStreamNoCRLF( + _header: tar.Headers, + stream: Readable, +): Promise { + const chai = await import('chai'); + const buf = await streamToBuffer(stream); + await chai.expect(buf.includes('\r\n')).to.be.false; } diff --git a/tests/test-data/projects/no-docker-compose/basic/Dockerfile b/tests/test-data/projects/no-docker-compose/basic/Dockerfile index 6c6f9c81..3bb4ed2f 100644 --- a/tests/test-data/projects/no-docker-compose/basic/Dockerfile +++ b/tests/test-data/projects/no-docker-compose/basic/Dockerfile @@ -1,4 +1,4 @@ FROM busybox -COPY ./src/start.sh /start.sh -RUN chmod a+x /start.sh -CMD ["/start.sh"] +COPY ./src /usr/src/ +RUN chmod a+x /usr/src/*.sh +CMD ["/usr/src/start.sh"] diff --git a/tests/test-data/projects/no-docker-compose/basic/src/windows-crlf.sh b/tests/test-data/projects/no-docker-compose/basic/src/windows-crlf.sh new file mode 100644 index 00000000..380ab640 --- /dev/null +++ b/tests/test-data/projects/no-docker-compose/basic/src/windows-crlf.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo 'this file was saved with Windows CRLF line endings' diff --git a/tests/utils/eol-conversion.spec.ts b/tests/utils/eol-conversion.spec.ts new file mode 100644 index 00000000..d039fa29 --- /dev/null +++ b/tests/utils/eol-conversion.spec.ts @@ -0,0 +1,55 @@ +/** + * @license + * Copyright 2020 Balena Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { expect } from 'chai'; + +import { convertEolInPlace } from '../../build/utils/eol-conversion'; + +describe('convertEolInPlace() function', function() { + it('should return expected values', () => { + // pairs of [given input, expected output] + const testVector = [ + ['', ''], + ['\r', '\r'], + ['\n', '\n'], + ['\r\r', '\r\r'], + ['\n\r', '\n\r'], + ['\r\n', '\n'], + ['\r\n\n', '\n\n'], + ['\r\n\r', '\n\r'], + ['\r\n\r\n', '\n\n'], + ['\r\n\n\r', '\n\n\r'], + ['abc\r\ndef\r\n', 'abc\ndef\n'], + ['abc\r\ndef\n\r', 'abc\ndef\n\r'], + ['abc\r\ndef\n', 'abc\ndef\n'], + ['abc\r\ndef\r', 'abc\ndef\r'], + ['abc\r\ndef', 'abc\ndef'], + ['\r\ndef\r\n', '\ndef\n'], + ['\rdef\r', '\rdef\r'], + ]; + const js = JSON.stringify; + + for (const [input, expected] of testVector) { + const result = convertEolInPlace(Buffer.from(input)); + const resultStr = result.toString(); + const msg = `input=${js(input)} result=${js(resultStr)} expected=${js( + expected, + )}`; + expect(resultStr).to.equal(expected, msg); + } + }); +});