diff --git a/doc/cli.markdown b/doc/cli.markdown index f02d228a..94186eaa 100644 --- a/doc/cli.markdown +++ b/doc/cli.markdown @@ -1805,6 +1805,7 @@ containers. The synchronization is only in one direction, from this machine to the device, and changes made on the device itself may be overwritten. This feature requires a device running supervisor version v9.7.0 or greater. +REGISTRY SECRETS The --registry-secrets option specifies a JSON or YAML file containing private Docker registry usernames and passwords to be used when pulling base images. Sample registry-secrets YAML file: @@ -1826,6 +1827,41 @@ If the --registry-secrets option is not specified, and a secrets.yml or secrets.json file exists in the balena directory (usually $HOME/.balena), this file will be used instead. +DOCKERIGNORE AND GITIGNORE FILES +By default, both '.dockerignore' and '.gitignore' files are taken into account +in order to prevent files from being sent to the balenaCloud builder or Docker +or balenaEngine (balenaOS device). + +However, this behavior has been DEPRECATED and will change in an upcoming major +version release. The --nogitignore (-G) option should be used to enable the new +behavior already now. This option will cause the CLI to: + +* Disregard all '.gitignore' files at the source directory and subdirectories, + and consider only the '.dockerignore' file (if any) at the source directory. +* Consequently, allow files to be sent to balenaCloud / Docker / balenaEngine + even if they are listed in '.gitignore' files (a longstanding feature request). +* Use a new '.dockerignore' parser and filter library that improves compatibility + with "docker build" and fixes several issues (mainly on Windows). +* Prevent a warning message from being printed. + +When --nogitignore (-G) is provided, a few "hardcoded" dockerignore patterns are +also used and "merged" (in memory) with the patterns found in the '.dockerignore' +file (if any), in the following order: + + **/.git + < user's patterns from the '.dockerignore' file, if any > + !**/.balena + !**/.resin + !**/Dockerfile + !**/Dockerfile.* + !**/docker-compose.yml + +If necessary, the effect of the '**/.git' pattern may be modified by adding +"counter patterns" to the '.dockerignore' file, for example '!service1/.git'. +For documentation on pattern format, see: +- https://docs.docker.com/engine/reference/builder/#dockerignore-file +- https://www.npmjs.com/package/@balena/dockerignore + Examples: $ balena push myApp @@ -1845,7 +1881,7 @@ Examples: #### --source, -s <source> -The source that should be sent to the balena builder to be built (defaults to the current directory) +Source directory to be sent to balenaCloud or balenaOS device (default: current working dir) #### --emulated, -e @@ -1909,6 +1945,12 @@ left hand side of the = character will be treated as the variable name. On Windows only, convert line endings from CRLF (Windows format) to LF (Unix format). Source files are not modified. +#### --nogitignore, -G + +Disregard all .gitignore files, and consider only the .dockerignore file (if any) +at the source directory. This will be the default behavior in an upcoming major +version release. For more information, see 'balena help push'. + # Settings ## settings @@ -1971,6 +2013,7 @@ found, it will look for a Dockerfile[.template] file (or alternative Dockerfile specified with the `--dockerfile` option), and if no dockerfile is found, it will try to generate one. +REGISTRY SECRETS The --registry-secrets option specifies a JSON or YAML file containing private Docker registry usernames and passwords to be used when pulling base images. Sample registry-secrets YAML file: @@ -1992,6 +2035,41 @@ If the --registry-secrets option is not specified, and a secrets.yml or secrets.json file exists in the balena directory (usually $HOME/.balena), this file will be used instead. +DOCKERIGNORE AND GITIGNORE FILES +By default, both '.dockerignore' and '.gitignore' files are taken into account +in order to prevent files from being sent to the balenaCloud builder or Docker +or balenaEngine (balenaOS device). + +However, this behavior has been DEPRECATED and will change in an upcoming major +version release. The --nogitignore (-G) option should be used to enable the new +behavior already now. This option will cause the CLI to: + +* Disregard all '.gitignore' files at the source directory and subdirectories, + and consider only the '.dockerignore' file (if any) at the source directory. +* Consequently, allow files to be sent to balenaCloud / Docker / balenaEngine + even if they are listed in '.gitignore' files (a longstanding feature request). +* Use a new '.dockerignore' parser and filter library that improves compatibility + with "docker build" and fixes several issues (mainly on Windows). +* Prevent a warning message from being printed. + +When --nogitignore (-G) is provided, a few "hardcoded" dockerignore patterns are +also used and "merged" (in memory) with the patterns found in the '.dockerignore' +file (if any), in the following order: + + **/.git + < user's patterns from the '.dockerignore' file, if any > + !**/.balena + !**/.resin + !**/Dockerfile + !**/Dockerfile.* + !**/docker-compose.yml + +If necessary, the effect of the '**/.git' pattern may be modified by adding +"counter patterns" to the '.dockerignore' file, for example '!service1/.git'. +For documentation on pattern format, see: +- https://docs.docker.com/engine/reference/builder/#dockerignore-file +- https://www.npmjs.com/package/@balena/dockerignore + Examples: $ balena build @@ -2032,6 +2110,12 @@ Alternative Dockerfile name/path, relative to the source folder Display full log output +#### --nogitignore, -G + +Disregard all .gitignore files, and consider only the .dockerignore file (if any) +at the source directory. This will be the default behavior in an upcoming major +version release. For more information, see 'balena help undefined'. + #### --noparent-check Disable project validation check of 'docker-compose.yml' file in parent folder @@ -2112,6 +2196,7 @@ To deploy to an app on which you're a collaborator, use When --build is used, all options supported by `balena build` are also supported by this command. +REGISTRY SECRETS The --registry-secrets option specifies a JSON or YAML file containing private Docker registry usernames and passwords to be used when pulling base images. Sample registry-secrets YAML file: @@ -2133,6 +2218,41 @@ If the --registry-secrets option is not specified, and a secrets.yml or secrets.json file exists in the balena directory (usually $HOME/.balena), this file will be used instead. +DOCKERIGNORE AND GITIGNORE FILES +By default, both '.dockerignore' and '.gitignore' files are taken into account +in order to prevent files from being sent to the balenaCloud builder or Docker +or balenaEngine (balenaOS device). + +However, this behavior has been DEPRECATED and will change in an upcoming major +version release. The --nogitignore (-G) option should be used to enable the new +behavior already now. This option will cause the CLI to: + +* Disregard all '.gitignore' files at the source directory and subdirectories, + and consider only the '.dockerignore' file (if any) at the source directory. +* Consequently, allow files to be sent to balenaCloud / Docker / balenaEngine + even if they are listed in '.gitignore' files (a longstanding feature request). +* Use a new '.dockerignore' parser and filter library that improves compatibility + with "docker build" and fixes several issues (mainly on Windows). +* Prevent a warning message from being printed. + +When --nogitignore (-G) is provided, a few "hardcoded" dockerignore patterns are +also used and "merged" (in memory) with the patterns found in the '.dockerignore' +file (if any), in the following order: + + **/.git + < user's patterns from the '.dockerignore' file, if any > + !**/.balena + !**/.resin + !**/Dockerfile + !**/Dockerfile.* + !**/docker-compose.yml + +If necessary, the effect of the '**/.git' pattern may be modified by adding +"counter patterns" to the '.dockerignore' file, for example '!service1/.git'. +For documentation on pattern format, see: +- https://docs.docker.com/engine/reference/builder/#dockerignore-file +- https://www.npmjs.com/package/@balena/dockerignore + Examples: $ balena deploy myApp @@ -2169,6 +2289,12 @@ Alternative Dockerfile name/path, relative to the source folder Display full log output +#### --nogitignore, -G + +Disregard all .gitignore files, and consider only the .dockerignore file (if any) +at the source directory. This will be the default behavior in an upcoming major +version release. For more information, see 'balena help undefined'. + #### --noparent-check Disable project validation check of 'docker-compose.yml' file in parent folder diff --git a/lib/actions/build.js b/lib/actions/build.js index 344c7f6f..139345a3 100644 --- a/lib/actions/build.js +++ b/lib/actions/build.js @@ -21,7 +21,7 @@ import * as Promise from 'bluebird'; import * as dockerUtils from '../utils/docker'; import * as compose from '../utils/compose'; -import { registrySecretsHelp } from '../utils/messages'; +import { dockerignoreHelp, registrySecretsHelp } from '../utils/messages'; import { getBalenaSdk } from '../utils/lazy'; /* @@ -62,6 +62,7 @@ const buildProject = function(docker, logger, composeOpts, opts) { composeOpts.inlineLogs, opts.convertEol, composeOpts.dockerfilePath, + composeOpts.nogitignore, ); }) .then(function() { @@ -95,6 +96,8 @@ will try to generate one. ${registrySecretsHelp} +${dockerignoreHelp} + Examples: $ balena build diff --git a/lib/actions/deploy.js b/lib/actions/deploy.js index d1e1f25a..4d9b50af 100644 --- a/lib/actions/deploy.js +++ b/lib/actions/deploy.js @@ -21,7 +21,7 @@ import * as Promise from 'bluebird'; import * as dockerUtils from '../utils/docker'; import * as compose from '../utils/compose'; -import { registrySecretsHelp } from '../utils/messages'; +import { dockerignoreHelp, registrySecretsHelp } from '../utils/messages'; import { ExpectedError } from '../errors'; import { getBalenaSdk, getChalk } from '../utils/lazy'; @@ -95,6 +95,7 @@ const deployProject = function(docker, logger, composeOpts, opts) { composeOpts.inlineLogs, opts.convertEol, composeOpts.dockerfilePath, + composeOpts.nogitignore, ) .then(builtImages => _.keyBy(builtImages, 'serviceName')); }) @@ -200,6 +201,8 @@ by this command. ${registrySecretsHelp} +${dockerignoreHelp} + Examples: $ balena deploy myApp diff --git a/lib/actions/push.ts b/lib/actions/push.ts index 8284e104..6c0ef8cd 100644 --- a/lib/actions/push.ts +++ b/lib/actions/push.ts @@ -20,7 +20,7 @@ import { stripIndent } from 'common-tags'; import { ExpectedError } from '../errors'; import { getBalenaSdk } from '../utils/lazy'; -import { registrySecretsHelp } from '../utils/messages'; +import { dockerignoreHelp, registrySecretsHelp } from '../utils/messages'; import { validateApplicationName, validateDotLocalUrl, @@ -109,6 +109,7 @@ export const push: CommandDefinition< nocache?: boolean; 'noparent-check'?: boolean; 'registry-secrets'?: string; + nogitignore?: boolean; nolive?: boolean; detached?: boolean; service?: string | string[]; @@ -148,6 +149,8 @@ export const push: CommandDefinition< ${registrySecretsHelp.split('\n').join('\n\t\t')} + ${dockerignoreHelp.split('\n').join('\n\t\t')} + Examples: $ balena push myApp @@ -168,7 +171,7 @@ export const push: CommandDefinition< signature: 'source', alias: 's', description: - 'The source that should be sent to the balena builder to be built (defaults to the current directory)', + 'Source directory to be sent to balenaCloud or balenaOS device (default: current working dir)', parameter: 'source', }, { @@ -259,6 +262,16 @@ export const push: CommandDefinition< Source files are not modified.`, boolean: true, }, + { + signature: 'nogitignore', + alias: 'G', + description: stripIndent` + Disregard all .gitignore files, and consider only the .dockerignore file (if any) + at the source directory. This will be the default behavior in an upcoming major + version release. For more information, see 'balena help push'. + `, + boolean: true, + }, ], async action(params, options) { const sdk = getBalenaSdk(); @@ -336,6 +349,7 @@ export const push: CommandDefinition< source, auth: token, baseUrl, + nogitignore: !!options.nogitignore, sdk, opts, }; @@ -359,6 +373,7 @@ export const push: CommandDefinition< dockerfilePath, registrySecrets, nocache: options.nocache || false, + nogitignore: options.nogitignore || false, noParentCheck: options['noparent-check'] || false, nolive: options.nolive || false, detached: options.detached || false, diff --git a/lib/preparser.ts b/lib/preparser.ts index 073f7aaa..c2fe5804 100644 --- a/lib/preparser.ts +++ b/lib/preparser.ts @@ -61,6 +61,9 @@ export async function routeCliFramework(argv: string[], options: AppOptions) { } } + const Logger = await import('./utils/logger'); + Logger.command = cmdSlice[0]; + const [isOclif, isTopic] = isOclifCommand(cmdSlice); if (isOclif) { @@ -68,6 +71,7 @@ export async function routeCliFramework(argv: string[], options: AppOptions) { if (isTopic) { // convert space-separated commands to oclif's topic:command syntax oclifArgs = [cmdSlice[0] + ':' + cmdSlice[1], ...cmdSlice.slice(2)]; + Logger.command = `${cmdSlice[0]} ${cmdSlice[1]}`; } if (process.env.DEBUG) { console.log( diff --git a/lib/utils/compose-types.d.ts b/lib/utils/compose-types.d.ts index 59ef838e..2b99e0cf 100644 --- a/lib/utils/compose-types.d.ts +++ b/lib/utils/compose-types.d.ts @@ -41,6 +41,7 @@ export interface ComposeProject { } interface TarDirectoryOptions { - preFinalizeCallback?: (pack: Pack) => void; convertEol?: boolean; + preFinalizeCallback?: (pack: Pack) => void | Promise; + nogitignore: boolean; } diff --git a/lib/utils/compose.js b/lib/utils/compose.js index 8ed8ef77..48d66c80 100644 --- a/lib/utils/compose.js +++ b/lib/utils/compose.js @@ -16,8 +16,11 @@ */ import * as Promise from 'bluebird'; +import { stripIndent } from 'common-tags'; import * as path from 'path'; + import { getBalenaSdk, getChalk } from './lazy'; +import { IgnoreFileType } from './ignore'; export const appendProjectOptions = opts => opts.concat([ @@ -31,6 +34,7 @@ export const appendProjectOptions = opts => ]); export function appendOptions(opts) { + const Logger = require('./logger'); return appendProjectOptions(opts).concat([ { signature: 'emulated', @@ -49,6 +53,16 @@ export function appendOptions(opts) { description: 'Display full log output', boolean: true, }, + { + signature: 'nogitignore', + description: stripIndent` + Disregard all .gitignore files, and consider only the .dockerignore file (if any) + at the source directory. This will be the default behavior in an upcoming major + version release. For more information, see 'balena help ${Logger.command}'. + `, + boolean: true, + alias: 'G', + }, { signature: 'noparent-check', description: @@ -83,6 +97,7 @@ export function generateOpts(options) { projectPath, inlineLogs: !!options.logs, dockerfilePath: options.dockerfile, + nogitignore: !!options.nogitignore, noParentCheck: options['noparent-check'], })); } @@ -132,15 +147,33 @@ export function createProject(composePath, composeStr, projectName = null) { } /** - * @param {string} dir - * @param {import('./compose-types').TarDirectoryOptions} [param] + * Create a tar stream out of the local filesystem at the given directory, + * while optionally applying file filters such as '.dockerignore' and + * optionally converting text file line endings (CRLF to LF). + * @param {string} dir Source directory + * @param {import('./compose-types').TarDirectoryOptions} param * @returns {Promise} */ -export const tarDirectory = function(dir, param) { - if (param == null) { - param = {}; +export function tarDirectory(dir, param) { + let { nogitignore = false } = param; + if (nogitignore) { + return Promise.resolve(require('./compose_ts').tarDirectory(dir, param)); + } else { + return originalTarDirectory(dir, param); } - let { preFinalizeCallback = null, convertEol = false } = param; +} + +/** + * @param {string} dir Source directory + * @param {import('./compose-types').TarDirectoryOptions} param + * @returns {Promise} + */ +function originalTarDirectory(dir, param) { + let { + preFinalizeCallback = null, + convertEol = false, + nogitignore = false, + } = param; if (convertEol == null) { convertEol = false; } @@ -149,6 +182,7 @@ export const tarDirectory = function(dir, param) { const klaw = require('klaw'); const fs = require('mz/fs'); const streamToPromise = require('stream-to-promise'); + const { printGitignoreWarn } = require('./compose_ts'); const { FileIgnorer } = require('./ignore'); const { toPosixPath } = require('resin-multibuild').PathUtils; let readFile; @@ -167,13 +201,24 @@ export const tarDirectory = function(dir, param) { const ignore = new FileIgnorer(dir); const pack = tar.pack(); + const ignoreFiles = {}; return getFiles() .each(function(file) { const type = ignore.getIgnoreFileType(path.relative(dir, file)); if (type != null) { + ignoreFiles[type] = ignoreFiles[type] || []; + ignoreFiles[type].push(path.resolve(dir, file)); return ignore.addIgnoreFile(file, type); } }) + .tap(() => { + if (!nogitignore) { + printGitignoreWarn( + (ignoreFiles[IgnoreFileType.DockerIgnore] || [])[0] || '', + ignoreFiles[IgnoreFileType.GitIgnore] || [], + ); + } + }) .filter(ignore.filter) .map(function(file) { const relPath = path.relative(path.resolve(dir), file); @@ -193,7 +238,7 @@ export const tarDirectory = function(dir, param) { pack.finalize(); return pack; }); -}; +} const truncateString = function(str, len) { if (str.length < len) { @@ -221,6 +266,7 @@ export function buildProject( inlineLogs, convertEol, dockerfilePath, + nogitignore, ) { const _ = require('lodash'); const humanize = require('humanize'); @@ -274,7 +320,7 @@ export function buildProject( .then(( needsQemu, // Tar up the directory, ready for the build stream ) => - tarDirectory(projectPath, { convertEol }) + tarDirectory(projectPath, { convertEol, nogitignore }) .then(tarStream => makeBuildTasks( composition, diff --git a/lib/utils/compose_ts.ts b/lib/utils/compose_ts.ts index 47847b8c..699e71eb 100644 --- a/lib/utils/compose_ts.ts +++ b/lib/utils/compose_ts.ts @@ -156,6 +156,93 @@ async function loadBuildMetatada( return [buildMetadata, metadataPath]; } +/** + * Create a tar stream out of the local filesystem at the given directory, + * while optionally applying file filters such as '.dockerignore' and + * optionally converting text file line endings (CRLF to LF). + * @param dir Source directory + * @param param Options + * @returns {Promise} + */ +export async function tarDirectory( + dir: string, + { + preFinalizeCallback, + convertEol = false, + nogitignore = false, + }: import('./compose-types').TarDirectoryOptions, +): Promise { + (await import('assert')).strict.strictEqual(nogitignore, true); + const { filterFilesWithDockerignore } = await import('./ignore'); + const { toPosixPath } = (await import('resin-multibuild')).PathUtils; + + let readFile: (file: string) => Promise; + if (process.platform === 'win32') { + const { readFileWithEolConversion } = require('./eol-conversion'); + readFile = file => readFileWithEolConversion(file, convertEol); + } else { + readFile = fs.readFile; + } + const pack = tar.pack(); + const fileStatsList = await filterFilesWithDockerignore(dir); + for (const fileStats of fileStatsList) { + pack.entry( + { + name: toPosixPath(fileStats.relPath), + size: fileStats.stats.size, + mode: fileStats.stats.mode, + }, + await readFile(fileStats.filePath), + ); + } + if (preFinalizeCallback) { + await preFinalizeCallback(pack); + } + pack.finalize(); + return pack; +} + +/** + * Print a deprecation warning if any '.gitignore' or '.dockerignore' file is + * found and the --nogitignore (-G) option has not been provided. + * @param dockerignoreFile Absolute path to a .dockerignore file + * @param gitignoreFiles Array of absolute paths to .gitginore files + */ +export function printGitignoreWarn( + dockerignoreFile: string, + gitignoreFiles: string[], +) { + const ignoreFiles = [dockerignoreFile, ...gitignoreFiles].filter(e => e); + if (ignoreFiles.length === 0) { + return; + } + const hr = + '-------------------------------------------------------------------------------'; + const msg = [' ', hr, 'Using file ignore patterns from:']; + msg.push(...ignoreFiles); + if (gitignoreFiles.length) { + msg.push(stripIndent` + balena CLI currently uses gitgnore and dockerignore files, but an upcoming major + version release will disregard gitignore files and use a dockerignore file only. + Use the --nogitignore (-G) option to enable the new behavior already now and + suppress this warning. For more information, see 'balena help ${Logger.command}'. + `); + msg.push(hr); + Logger.getLogger().logWarn(msg.join('\n')); + } else if (dockerignoreFile && process.platform === 'win32') { + msg.push(stripIndent` + Use the --nogitignore (-G) option to suppress this warning and enable the use + of a better dockerignore parser and filter library that fixes several issues + on Windows and improves compatibility with "docker build", but which may also + cause a different set of files to be filtered out (because of the bug fixes). + The --nogitignore option will be the default behavior in an upcoming balena CLI + major version release. For more information, see 'balena help ${Logger.command}'. + `); + msg.push(hr); + Logger.getLogger().logWarn(msg.join('\n')); + } +} + /** * Check whether the "build secrets" feature is being used and, if so, * verify that the target docker daemon is balenaEngine. If the diff --git a/lib/utils/device/deploy.ts b/lib/utils/device/deploy.ts index 322209d3..e8c7bbd8 100644 --- a/lib/utils/device/deploy.ts +++ b/lib/utils/device/deploy.ts @@ -56,6 +56,7 @@ export interface DeviceDeployOptions { dockerfilePath?: string; registrySecrets: RegistrySecrets; nocache: boolean; + nogitignore: boolean; noParentCheck: boolean; nolive: boolean; detached: boolean; @@ -186,6 +187,7 @@ export async function deployToDevice(opts: DeviceDeployOptions): Promise { globalLogger.logDebug('Tarring all non-ignored files...'); const tarStream = await tarDirectory(opts.source, { convertEol: opts.convertEol, + nogitignore: opts.nogitignore, }); // Try to detect the device information @@ -400,7 +402,10 @@ export async function rebuildSingleTask( } }; - const tarStream = await tarDirectory(source); + const tarStream = await tarDirectory(source, { + convertEol: opts.convertEol, + nogitignore: opts.nogitignore, + }); const task = _.find( await makeBuildTasks( diff --git a/lib/utils/ignore.ts b/lib/utils/ignore.ts index 6e5b8822..15113a23 100644 --- a/lib/utils/ignore.ts +++ b/lib/utils/ignore.ts @@ -22,6 +22,8 @@ import * as MultiBuild from 'resin-multibuild'; import dockerIgnore = require('@zeit/dockerignore'); import ignore from 'ignore'; +import { ExpectedError } from '../errors'; + const { toPosixPath } = MultiBuild.PathUtils; export enum IgnoreFileType { @@ -182,3 +184,92 @@ export class FileIgnorer { return !/^\.\.\//.test(path.posix.relative(path1, path2)); } } + +interface FileStats { + filePath: string; + relPath: string; + stats: fs.Stats; +} + +/** + * Create a list of files (FileStats[]) for the filesystem subtree rooted at + * projectDir, listing each file with both a full path and a relative path, + * but excluding entries for directories themselves. + * @param projectDir Source directory (root of subtree to be listed) + * @param dir Used for recursive calls only (omit on first function call) + */ +async function listFiles( + projectDir: string, + dir: string = projectDir, +): Promise { + const files: FileStats[] = []; + const dirEntries = await fs.readdir(dir); + await Promise.all( + dirEntries.map(async entry => { + const filePath = path.join(dir, entry); + const stats = await fs.stat(filePath); + if (stats.isDirectory()) { + files.push(...(await listFiles(projectDir, filePath))); + } else if (stats.isFile()) { + files.push({ + filePath, + relPath: path.relative(projectDir, filePath), + stats, + }); + } + }), + ); + return files; +} + +/** + * Return the contents of a .dockerignore file at projectDir, as a string. + * Return an empty string if a .dockerignore file does not exist. + * @param projectDir Source directory + * @returns Contents of the .dockerignore file, as a UTF-8 string + */ +async function readDockerIgnoreFile(projectDir: string): Promise { + const dockerIgnorePath = path.join(projectDir, '.dockerignore'); + let dockerIgnoreStr = ''; + try { + dockerIgnoreStr = await fs.readFile(dockerIgnorePath, 'utf8'); + } catch (err) { + if (err.code !== 'ENOENT') { + throw new ExpectedError( + `Error reading file "${dockerIgnorePath}": ${err.message}`, + ); + } + } + return dockerIgnoreStr; +} + +/** + * Create a list of files (FileStats[]) for the filesystem subtree rooted at + * projectDir, filtered against a .dockerignore file (if any) also at projectDir, + * plus a few hardcoded dockerignore patterns. + * @param projectDir Source directory to + */ +export async function filterFilesWithDockerignore( + projectDir: string, +): Promise { + // path.resolve() also converts forward slashes to backslashes on Windows + projectDir = path.resolve(projectDir); + const dockerIgnoreStr = await readDockerIgnoreFile(projectDir); + const $dockerIgnore = (await import('@balena/dockerignore')).default; + const ig = $dockerIgnore({ ignorecase: false }); + + ig.add(['**/.git']); + if (dockerIgnoreStr) { + ig.add(dockerIgnoreStr); + } + ig.add([ + '!**/.balena', + '!**/.resin', + '!**/Dockerfile', + '!**/Dockerfile.*', + '!**/docker-compose.yml', + ]); + + const files = await listFiles(projectDir); + return files.filter((file: FileStats) => !ig.ignores(file.relPath)); +} diff --git a/lib/utils/logger.ts b/lib/utils/logger.ts index 27c7cfe9..461182df 100644 --- a/lib/utils/logger.ts +++ b/lib/utils/logger.ts @@ -39,6 +39,8 @@ enum Level { */ class Logger { public static readonly Level = Level; + // `Logger.command` is currently set in `preparser.ts` + public static command: string; // CLI cmd, e.g. 'push', 'env add', ... public streams: { build: NodeJS.ReadWriteStream; diff --git a/lib/utils/messages.ts b/lib/utils/messages.ts index ae909026..18298dc6 100644 --- a/lib/utils/messages.ts +++ b/lib/utils/messages.ts @@ -1,3 +1,20 @@ +/** + * @license + * Copyright 2017-2020 Balena Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + const DEBUG_MODE = !!process.env.DEBUG; export const reachingOut = `\ @@ -28,6 +45,7 @@ export const balenaAsciiArt = `\ `; export const registrySecretsHelp = `\ +REGISTRY SECRETS The --registry-secrets option specifies a JSON or YAML file containing private Docker registry usernames and passwords to be used when pulling base images. Sample registry-secrets YAML file: @@ -48,3 +66,39 @@ check: https://github.com/balena-io-playground/sample-gcr-registry-secrets If the --registry-secrets option is not specified, and a secrets.yml or secrets.json file exists in the balena directory (usually $HOME/.balena), this file will be used instead.`; + +export const dockerignoreHelp = `\ +DOCKERIGNORE AND GITIGNORE FILES +By default, both '.dockerignore' and '.gitignore' files are taken into account +in order to prevent files from being sent to the balenaCloud builder or Docker +or balenaEngine (balenaOS device). + +However, this behavior has been DEPRECATED and will change in an upcoming major +version release. The --nogitignore (-G) option should be used to enable the new +behavior already now. This option will cause the CLI to: + +* Disregard all '.gitignore' files at the source directory and subdirectories, + and consider only the '.dockerignore' file (if any) at the source directory. +* Consequently, allow files to be sent to balenaCloud / Docker / balenaEngine + even if they are listed in '.gitignore' files (a longstanding feature request). +* Use a new '.dockerignore' parser and filter library that improves compatibility + with "docker build" and fixes several issues (mainly on Windows). +* Prevent a warning message from being printed. + +When --nogitignore (-G) is provided, a few "hardcoded" dockerignore patterns are +also used and "merged" (in memory) with the patterns found in the '.dockerignore' +file (if any), in the following order: + + **/.git + < user's patterns from the '.dockerignore' file, if any > + !**/.balena + !**/.resin + !**/Dockerfile + !**/Dockerfile.* + !**/docker-compose.yml + +If necessary, the effect of the '**/.git' pattern may be modified by adding +"counter patterns" to the '.dockerignore' file, for example '!service1/.git'. +For documentation on pattern format, see: +- https://docs.docker.com/engine/reference/builder/#dockerignore-file +- https://www.npmjs.com/package/@balena/dockerignore`; diff --git a/lib/utils/remote-build.ts b/lib/utils/remote-build.ts index 1b573ef1..fbe73346 100644 --- a/lib/utils/remote-build.ts +++ b/lib/utils/remote-build.ts @@ -51,6 +51,7 @@ export interface RemoteBuild { source: string; auth: string; baseUrl: string; + nogitignore: boolean; opts: BuildOpts; sdk: BalenaSDK; @@ -302,6 +303,7 @@ async function getTarStream(build: RemoteBuild): Promise { return await tarDirectory(path.resolve(build.source), { preFinalizeCallback: preFinalizeCb, convertEol: build.opts.convertEol, + nogitignore: build.nogitignore, }); } finally { tarSpinner.stop(); diff --git a/npm-shrinkwrap.json b/npm-shrinkwrap.json index 92a245d2..a6b40909 100644 --- a/npm-shrinkwrap.json +++ b/npm-shrinkwrap.json @@ -104,6 +104,11 @@ } } }, + "@balena/dockerignore": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@balena/dockerignore/-/dockerignore-1.0.2.tgz", + "integrity": "sha512-wMue2Sy4GAVTk6Ic4tJVcnfdau+gx2EnG7S+uAEe+TWJFqE4YoWN4/H8MSLj4eYJKxGg26lZwboEniNiNwZQ6Q==" + }, "@balena/lint": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/@balena/lint/-/lint-4.1.1.tgz", diff --git a/package.json b/package.json index f5fbfb76..4f07f7a4 100644 --- a/package.json +++ b/package.json @@ -164,6 +164,7 @@ "typescript": "^3.8.3" }, "dependencies": { + "@balena/dockerignore": "^1.0.2", "@oclif/command": "^1.5.19", "@resin.io/valid-email": "^0.1.0", "@sentry/node": "^5.13.2", diff --git a/tests/commands/build.spec.ts b/tests/commands/build.spec.ts index e38cba88..bfcc5700 100644 --- a/tests/commands/build.spec.ts +++ b/tests/commands/build.spec.ts @@ -25,14 +25,13 @@ import { fs } from 'mz'; import * as path from 'path'; import { BalenaAPIMock } from '../balena-api-mock'; +import { expectStreamNoCRLF, testDockerBuildStream } from '../docker-build'; +import { DockerMock, dockerResponsePath } from '../docker-mock'; +import { cleanOutput, runCommand } from '../helpers'; import { ExpectedTarStreamFiles, ExpectedTarStreamFilesByService, - expectStreamNoCRLF, - testDockerBuildStream, -} from '../docker-build'; -import { DockerMock, dockerResponsePath } from '../docker-mock'; -import { cleanOutput, runCommand } from '../helpers'; +} from '../projects'; const repoPath = path.normalize(path.join(__dirname, '..', '..')); const projectsPath = path.join(repoPath, 'tests', 'test-data', 'projects'); @@ -53,6 +52,15 @@ const commonQueryParams = [ ['labels', ''], ]; +const commonComposeQueryParams = [ + ['t', '${tag}'], + [ + 'buildargs', + '{"MY_VAR_1":"This is a variable","MY_VAR_2":"Also a variable"}', + ], + ['labels', ''], +]; + describe('balena build', function() { let api: BalenaAPIMock; let docker: DockerMock; @@ -104,7 +112,7 @@ describe('balena build', function() { } docker.expectGetInfo({}); await testDockerBuildStream({ - commandLine: `build ${projectPath} --deviceType nuc --arch amd64`, + commandLine: `build ${projectPath} --deviceType nuc --arch amd64 -G`, dockerMock: docker, expectedFilesByService: { main: expectedFiles }, expectedQueryParamsByService: { main: commonQueryParams }, @@ -178,7 +186,7 @@ describe('balena build', function() { mock.reRequire('../../build/utils/qemu'); docker.expectGetInfo({ OperatingSystem: 'balenaOS 2.44.0+rev1' }); await testDockerBuildStream({ - commandLine: `build ${projectPath} --emulated --deviceType ${deviceType} --arch ${arch}`, + commandLine: `build ${projectPath} --emulated --deviceType ${deviceType} --arch ${arch} --nogitignore`, dockerMock: docker, expectedFilesByService: { main: expectedFiles }, expectedQueryParamsByService: { main: commonQueryParams }, @@ -273,8 +281,15 @@ describe('balena build', function() { 'utf8', ); const expectedQueryParamsByService = { - service1: commonQueryParams, - service2: [...commonQueryParams, ['dockerfile', 'Dockerfile-alt']], + service1: [ + ['t', '${tag}'], + [ + 'buildargs', + '{"MY_VAR_1":"This is a variable","MY_VAR_2":"Also a variable","SERVICE1_VAR":"This is a service specific variable"}', + ], + ['labels', ''], + ], + service2: [...commonComposeQueryParams, ['dockerfile', 'Dockerfile-alt']], }; const expectedResponseLines: string[] = [ ...commonResponseLines[responseFilename], @@ -292,7 +307,7 @@ describe('balena build', function() { } docker.expectGetInfo({}); await testDockerBuildStream({ - commandLine: `build ${projectPath} --deviceType nuc --arch amd64 --convert-eol`, + commandLine: `build ${projectPath} --deviceType nuc --arch amd64 --convert-eol -G`, dockerMock: docker, expectedFilesByService, expectedQueryParamsByService, diff --git a/tests/commands/deploy.spec.ts b/tests/commands/deploy.spec.ts index f260d020..1d028366 100644 --- a/tests/commands/deploy.spec.ts +++ b/tests/commands/deploy.spec.ts @@ -23,9 +23,10 @@ import { fs } from 'mz'; import * as path from 'path'; import { BalenaAPIMock } from '../balena-api-mock'; -import { ExpectedTarStreamFiles, testDockerBuildStream } from '../docker-build'; +import { testDockerBuildStream } from '../docker-build'; import { DockerMock, dockerResponsePath } from '../docker-mock'; import { cleanOutput, runCommand } from '../helpers'; +import { ExpectedTarStreamFiles } from '../projects'; const repoPath = path.normalize(path.join(__dirname, '..', '..')); const projectsPath = path.join(repoPath, 'tests', 'test-data', 'projects'); @@ -119,7 +120,7 @@ describe('balena deploy', function() { } await testDockerBuildStream({ - commandLine: `deploy testApp --build --source ${projectPath}`, + commandLine: `deploy testApp --build --source ${projectPath} -G`, dockerMock: docker, expectedFilesByService: { main: expectedFiles }, expectedQueryParamsByService: { main: commonQueryParams }, diff --git a/tests/commands/push.spec.ts b/tests/commands/push.spec.ts index 169be0b8..a0b8485a 100644 --- a/tests/commands/push.spec.ts +++ b/tests/commands/push.spec.ts @@ -24,12 +24,13 @@ import * as path from 'path'; import { BalenaAPIMock } from '../balena-api-mock'; import { BuilderMock, builderResponsePath } from '../builder-mock'; -import { - ExpectedTarStreamFiles, - expectStreamNoCRLF, - testPushBuildStream, -} from '../docker-build'; +import { expectStreamNoCRLF, testPushBuildStream } from '../docker-build'; import { cleanOutput, runCommand } from '../helpers'; +import { + addRegSecretsEntries, + ExpectedTarStreamFiles, + setupDockerignoreTestData, +} from '../projects'; const repoPath = path.normalize(path.join(__dirname, '..', '..')); const projectsPath = path.join(repoPath, 'tests', 'test-data', 'projects'); @@ -76,6 +77,8 @@ const commonQueryParams = [ ['headless', 'false'], ]; +const itSkipWindows = process.platform === 'win32' ? it.skip : it; + describe('balena push', function() { let api: BalenaAPIMock; let builder: BuilderMock; @@ -95,6 +98,14 @@ describe('balena push', function() { builder.done(); }); + this.beforeAll(async () => { + await setupDockerignoreTestData(); + }); + + this.afterAll(async () => { + await setupDockerignoreTestData({ cleanup: true }); + }); + it('should create the expected tar stream (single container)', async () => { const projectPath = path.join(projectsPath, 'no-docker-compose', 'basic'); const expectedFiles: ExpectedTarStreamFiles = { @@ -103,6 +114,7 @@ describe('balena push', function() { Dockerfile: { fileSize: 88, type: 'file' }, 'Dockerfile-alt': { fileSize: 30, type: 'file' }, }; + const regSecretsPath = await addRegSecretsEntries(expectedFiles); const responseFilename = 'build-POST-v3.json'; const responseBody = await fs.readFile( path.join(builderResponsePath, responseFilename), @@ -122,7 +134,7 @@ describe('balena push', function() { await testPushBuildStream({ builderMock: builder, - commandLine: `push testApp --source ${projectPath}`, + commandLine: `push testApp --source ${projectPath} -R ${regSecretsPath} -G`, expectedFiles, expectedQueryParams: commonQueryParams, expectedResponseLines, @@ -140,6 +152,7 @@ describe('balena push', function() { Dockerfile: { fileSize: 88, type: 'file' }, 'Dockerfile-alt': { fileSize: 30, type: 'file' }, }; + const regSecretsPath = await addRegSecretsEntries(expectedFiles); const responseFilename = 'build-POST-v3.json'; const responseBody = await fs.readFile( path.join(builderResponsePath, responseFilename), @@ -151,7 +164,7 @@ describe('balena push', function() { await testPushBuildStream({ builderMock: builder, - commandLine: `push testApp --source ${projectPath} --dockerfile Dockerfile-alt`, + commandLine: `push testApp -s ${projectPath} -R ${regSecretsPath} --dockerfile Dockerfile-alt --nogitignore`, expectedFiles, expectedQueryParams, expectedResponseLines: commonResponseLines[responseFilename], @@ -173,6 +186,7 @@ describe('balena push', function() { Dockerfile: { fileSize: 88, type: 'file' }, 'Dockerfile-alt': { fileSize: 30, type: 'file' }, }; + const regSecretsPath = await addRegSecretsEntries(expectedFiles); const responseFilename = 'build-POST-v3.json'; const responseBody = await fs.readFile( path.join(builderResponsePath, responseFilename), @@ -191,7 +205,182 @@ describe('balena push', function() { await testPushBuildStream({ builderMock: builder, - commandLine: `push testApp --source ${projectPath} --convert-eol`, + commandLine: `push testApp -s ${projectPath} -R ${regSecretsPath} -l`, + expectedFiles, + expectedQueryParams: commonQueryParams, + expectedResponseLines, + projectPath, + responseBody, + responseCode: 200, + }); + }); + + // Skip Windows because the old tarDirectory() implementation (still used when + // '--nogitignore' is not provided) uses the old `zeit/dockerignore` npm package + // that is broken on Windows (reason why we created `@balena/dockerignore`). + itSkipWindows( + 'should create the expected tar stream (single container, with gitignore)', + async () => { + const projectPath = path.join( + projectsPath, + 'no-docker-compose', + 'dockerignore1', + ); + const expectedFiles: ExpectedTarStreamFiles = { + '.balena/balena.yml': { fileSize: 12, type: 'file' }, + '.dockerignore': { fileSize: 438, type: 'file' }, + '.gitignore': { fileSize: 20, type: 'file' }, + '.git/bar.txt': { fileSize: 4, type: 'file' }, + '.git/foo.txt': { fileSize: 4, type: 'file' }, + 'c.txt': { fileSize: 1, type: 'file' }, + Dockerfile: { fileSize: 13, type: 'file' }, + 'src/.balena/balena.yml': { fileSize: 16, type: 'file' }, + 'src/.gitignore': { fileSize: 10, type: 'file' }, + 'vendor/.git/vendor-git-contents': { fileSize: 20, type: 'file' }, + }; + const regSecretsPath = await addRegSecretsEntries(expectedFiles); + const responseFilename = 'build-POST-v3.json'; + const responseBody = await fs.readFile( + path.join(builderResponsePath, responseFilename), + 'utf8', + ); + const expectedResponseLines = [ + '[Warn] Using file ignore patterns from:', + `[Warn] ${path.join(projectPath, '.dockerignore')}`, + `[Warn] ${path.join(projectPath, '.gitignore')}`, + `[Warn] ${path.join(projectPath, 'src', '.gitignore')}`, + '[Warn] balena CLI currently uses gitgnore and dockerignore files, but an upcoming major', + '[Warn] version release will disregard gitignore files and use a dockerignore file only.', + '[Warn] Use the --nogitignore (-G) option to enable the new behavior already now and', + "[Warn] suppress this warning. For more information, see 'balena help push'.", + ...commonResponseLines[responseFilename], + ]; + + await testPushBuildStream({ + builderMock: builder, + commandLine: `push testApp -s ${projectPath} -R ${regSecretsPath} -l`, + expectedFiles, + expectedQueryParams: commonQueryParams, + expectedResponseLines, + projectPath, + responseBody, + responseCode: 200, + }); + }, + ); + + it('should create the expected tar stream (single container, --nogitignore)', async () => { + const projectPath = path.join( + projectsPath, + 'no-docker-compose', + 'dockerignore1', + ); + const expectedFiles: ExpectedTarStreamFiles = { + '.balena/balena.yml': { fileSize: 12, type: 'file' }, + '.dockerignore': { fileSize: 438, type: 'file' }, + '.gitignore': { fileSize: 20, type: 'file' }, + '.git/foo.txt': { fileSize: 4, type: 'file' }, + 'a.txt': { fileSize: 1, type: 'file' }, + 'c.txt': { fileSize: 1, type: 'file' }, + Dockerfile: { fileSize: 13, type: 'file' }, + 'src/.balena/balena.yml': { fileSize: 16, type: 'file' }, + 'src/.gitignore': { fileSize: 10, type: 'file' }, + 'src/src-a.txt': { fileSize: 5, type: 'file' }, + 'src/src-c.txt': { fileSize: 5, type: 'file' }, + 'vendor/.git/vendor-git-contents': { fileSize: 20, type: 'file' }, + }; + const regSecretsPath = await addRegSecretsEntries(expectedFiles); + const responseFilename = 'build-POST-v3.json'; + const responseBody = await fs.readFile( + path.join(builderResponsePath, responseFilename), + 'utf8', + ); + + await testPushBuildStream({ + builderMock: builder, + commandLine: `push testApp -s ${projectPath} -R ${regSecretsPath} -l -G`, + expectedFiles, + expectedQueryParams: commonQueryParams, + expectedResponseLines: commonResponseLines[responseFilename], + projectPath, + responseBody, + responseCode: 200, + }); + }); + + it('should create the expected tar stream (single container, symbolic links, --nogitignore)', async () => { + const projectPath = path.join( + projectsPath, + 'no-docker-compose', + 'dockerignore2', + ); + const expectedFiles: ExpectedTarStreamFiles = { + '.dockerignore': { fileSize: 34, type: 'file' }, + 'b.txt': { fileSize: 1, type: 'file' }, + Dockerfile: { fileSize: 13, type: 'file' }, + 'src/src-b.txt': { fileSize: 5, type: 'file' }, + 'symlink-a.txt': { fileSize: 5, type: 'file' }, + }; + const regSecretsPath = await addRegSecretsEntries(expectedFiles); + const responseFilename = 'build-POST-v3.json'; + const responseBody = await fs.readFile( + path.join(builderResponsePath, responseFilename), + 'utf8', + ); + + await testPushBuildStream({ + builderMock: builder, + commandLine: `push testApp -s ${projectPath} -R ${regSecretsPath} -l -G`, + expectedFiles, + expectedQueryParams: commonQueryParams, + expectedResponseLines: commonResponseLines[responseFilename], + projectPath, + responseBody, + responseCode: 200, + }); + }); + + it('should create the expected tar stream (single container, dockerignore warn)', async () => { + const projectPath = path.join( + projectsPath, + 'no-docker-compose', + 'dockerignore2', + ); + const expectedFiles: ExpectedTarStreamFiles = { + '.dockerignore': { fileSize: 34, type: 'file' }, + 'b.txt': { fileSize: 1, type: 'file' }, + Dockerfile: { fileSize: 13, type: 'file' }, + 'src/src-b.txt': { fileSize: 5, type: 'file' }, + 'symlink-a.txt': { fileSize: 5, type: 'file' }, + }; + if (isWindows) { + // this test uses the old tarDirectory implementation, which uses + // the zeit/dockerignore library that has bugs on Windows + expectedFiles['src/src-a.txt'] = { fileSize: 5, type: 'file' }; + } + const regSecretsPath = await addRegSecretsEntries(expectedFiles); + const responseFilename = 'build-POST-v3.json'; + const responseBody = await fs.readFile( + path.join(builderResponsePath, responseFilename), + 'utf8', + ); + const expectedResponseLines = isWindows + ? [ + '[Warn] Using file ignore patterns from:', + `[Warn] ${path.join(projectPath, '.dockerignore')}`, + '[Warn] Use the --nogitignore (-G) option to suppress this warning and enable the use', + '[Warn] of a better dockerignore parser and filter library that fixes several issues', + '[Warn] on Windows and improves compatibility with "docker build", but which may also', + '[Warn] cause a different set of files to be filtered out (because of the bug fixes).', + '[Warn] The --nogitignore option will be the default behavior in an upcoming balena CLI', + "[Warn] major version release. For more information, see 'balena help push'.", + ...commonResponseLines[responseFilename], + ] + : commonResponseLines[responseFilename]; + + await testPushBuildStream({ + builderMock: builder, + commandLine: `push testApp -s ${projectPath} -R ${regSecretsPath} -l`, expectedFiles, expectedQueryParams: commonQueryParams, expectedResponseLines, @@ -204,6 +393,8 @@ describe('balena push', function() { it('should create the expected tar stream (docker-compose)', async () => { const projectPath = path.join(projectsPath, 'docker-compose', 'basic'); const expectedFiles: ExpectedTarStreamFiles = { + '.balena/balena.yml': { fileSize: 197, type: 'file' }, + '.dockerignore': { fileSize: 22, type: 'file' }, 'docker-compose.yml': { fileSize: 245, type: 'file' }, 'service1/Dockerfile.template': { fileSize: 144, type: 'file' }, 'service1/file1.sh': { fileSize: 12, type: 'file' }, @@ -214,6 +405,7 @@ describe('balena push', function() { type: 'file', }, }; + const regSecretsPath = await addRegSecretsEntries(expectedFiles); const responseFilename = 'build-POST-v3.json'; const responseBody = await fs.readFile( path.join(builderResponsePath, responseFilename), @@ -234,7 +426,7 @@ describe('balena push', function() { await testPushBuildStream({ builderMock: builder, - commandLine: `push testApp --source ${projectPath} --convert-eol`, + commandLine: `push testApp -s ${projectPath} -R ${regSecretsPath} -l -G`, expectedFiles, expectedQueryParams: commonQueryParams, expectedResponseLines, @@ -258,7 +450,7 @@ describe('balena push: project validation', function() { ]; const { out, err } = await runCommand( - `push testApp --source ${projectPath}`, + `push testApp --source ${projectPath} --nogitignore`, ); expect( cleanOutput(err).map(line => line.replace(/\s{2,}/g, ' ')), diff --git a/tests/docker-build.ts b/tests/docker-build.ts index 94cdfce6..f7422507 100644 --- a/tests/docker-build.ts +++ b/tests/docker-build.ts @@ -29,25 +29,11 @@ import { URL } from 'url'; import { BuilderMock } from './builder-mock'; import { DockerMock } from './docker-mock'; import { cleanOutput, fillTemplateArray, runCommand } from './helpers'; - -export interface ExpectedTarStreamFile { - contents?: string; - fileSize: number; - testStream?: ( - header: tar.Headers, - stream: Readable, - expected?: ExpectedTarStreamFile, - ) => Promise; - type: tar.Headers['type']; -} - -export interface ExpectedTarStreamFiles { - [filePath: string]: ExpectedTarStreamFile; -} - -export interface ExpectedTarStreamFilesByService { - [service: string]: ExpectedTarStreamFiles; -} +import { + ExpectedTarStreamFile, + ExpectedTarStreamFiles, + ExpectedTarStreamFilesByService, +} from './projects'; /** * Run a few chai.expect() test assertions on a tar stream/buffer produced by @@ -77,21 +63,16 @@ export async function inspectTarStream( 'entry', async (header: tar.Headers, stream: Readable, next: tar.Callback) => { try { - // TODO: test the .balena folder instead of ignoring it - if (header.name.startsWith('.balena/')) { - stream.resume(); + expect(foundFiles).to.not.have.property(header.name); + foundFiles[header.name] = { + fileSize: header.size || 0, + type: header.type, + }; + const expected = expectedFiles[header.name]; + if (expected && expected.testStream) { + await expected.testStream(header, stream, expected); } else { - expect(foundFiles).to.not.have.property(header.name); - foundFiles[header.name] = { - fileSize: header.size || 0, - type: header.type, - }; - const expected = expectedFiles[header.name]; - if (expected && expected.testStream) { - await expected.testStream(header, stream, expected); - } else { - await defaultTestStream(header, stream, expected, projectPath); - } + await defaultTestStream(header, stream, expected, projectPath); } } catch (err) { reject(err); @@ -122,6 +103,11 @@ async function defaultTestStream( if (expected?.contents) { expectedContents = Buffer.from(expected.contents); } + if (header.name === '.balena/registry-secrets.json') { + expectedContents = await fs.readFile( + path.join(__dirname, 'test-data', 'projects', 'registry-secrets.json'), + ); + } const [buf, buf2] = await Promise.all([ streamToBuffer(stream), expectedContents || diff --git a/tests/projects.ts b/tests/projects.ts new file mode 100644 index 00000000..3913ed86 --- /dev/null +++ b/tests/projects.ts @@ -0,0 +1,85 @@ +/** + * @license + * Copyright 2020 Balena Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import * as fs from 'fs'; +import * as path from 'path'; +import { Headers } from 'tar-stream'; +import { promisify } from 'util'; + +const statAsync = promisify(fs.stat); + +export interface ExpectedTarStreamFile { + contents?: string; + fileSize: number; + testStream?: ( + header: Headers, + stream: import('stream').Readable, + expected?: ExpectedTarStreamFile, + ) => Promise; + type: Headers['type']; +} + +export interface ExpectedTarStreamFiles { + [filePath: string]: ExpectedTarStreamFile; +} + +export interface ExpectedTarStreamFilesByService { + [service: string]: ExpectedTarStreamFiles; +} + +export const repoPath = path.normalize(path.join(__dirname, '..')); +export const projectsPath = path.join( + repoPath, + 'tests', + 'test-data', + 'projects', +); + +export async function setupDockerignoreTestData({ cleanup = false } = {}) { + const { copy, remove } = await import('fs-extra'); + const dockerignoreProjDir = path.join( + __dirname, + 'test-data', + 'projects', + 'no-docker-compose', + 'dockerignore1', + ); + const subdirs = ['', 'vendor']; + for (const subdir of subdirs) { + // A git repo cannot store a '.git' subfolder, even under tests/test-data/, + // so we store a 'dot.git' folder instead, and copy it as '.git' before + // running the tests. (Interestingly, 'git status' also ignores the '.git' + // folder, and shows a "clean repo" even after this copy is executed.) + const aliasDir = path.join(dockerignoreProjDir, subdir, 'dot.git'); + const gitDir = path.join(dockerignoreProjDir, subdir, '.git'); + await remove(gitDir); + if (!cleanup) { + await copy(aliasDir, gitDir); + } + } +} + +export async function addRegSecretsEntries( + expectedFiles: ExpectedTarStreamFiles, +): Promise { + const regSecretsPath = path.join(projectsPath, 'registry-secrets.json'); + expectedFiles['.balena/registry-secrets.json'] = { + fileSize: (await statAsync(regSecretsPath)).size, + type: 'file', + }; + return regSecretsPath; +} diff --git a/tests/test-data/projects/docker-compose/basic/.balena/balena.yml b/tests/test-data/projects/docker-compose/basic/.balena/balena.yml new file mode 100644 index 00000000..ef6db889 --- /dev/null +++ b/tests/test-data/projects/docker-compose/basic/.balena/balena.yml @@ -0,0 +1,7 @@ +build-variables: + global: + - MY_VAR_1=This is a variable + - MY_VAR_2=Also a variable + services: + service1: + - SERVICE1_VAR=This is a service specific variable diff --git a/tests/test-data/projects/docker-compose/basic/.dockerignore b/tests/test-data/projects/docker-compose/basic/.dockerignore new file mode 100644 index 00000000..9a30e714 --- /dev/null +++ b/tests/test-data/projects/docker-compose/basic/.dockerignore @@ -0,0 +1 @@ +service1/test-ignore* diff --git a/tests/test-data/projects/docker-compose/basic/service1/test-ignore.txt b/tests/test-data/projects/docker-compose/basic/service1/test-ignore.txt new file mode 100644 index 00000000..47e97a99 --- /dev/null +++ b/tests/test-data/projects/docker-compose/basic/service1/test-ignore.txt @@ -0,0 +1 @@ +test-ignore diff --git a/tests/test-data/projects/no-docker-compose/dockerignore1/.balena/balena.yml b/tests/test-data/projects/no-docker-compose/dockerignore1/.balena/balena.yml new file mode 100644 index 00000000..862fc2ec --- /dev/null +++ b/tests/test-data/projects/no-docker-compose/dockerignore1/.balena/balena.yml @@ -0,0 +1 @@ +"balena.yml" \ No newline at end of file diff --git a/tests/test-data/projects/no-docker-compose/dockerignore1/.dockerignore b/tests/test-data/projects/no-docker-compose/dockerignore1/.dockerignore new file mode 100644 index 00000000..6e234400 --- /dev/null +++ b/tests/test-data/projects/no-docker-compose/dockerignore1/.dockerignore @@ -0,0 +1,19 @@ +# Note that the CLI "hardcodes" some dockerignore patterns in its source code, +# which get "merged" with the user's patterns in their .dockerginore file as follows: +# +# **/.git +# +# !**/.balena +# !**/.resin +# !**/Dockerfile +# !**/Dockerfile.* +# !**/docker-compose.yml +# +b.txt +Dockerfile +registry-secrets.json +src/*b.txt +**/.balena +**/dot.git +!.git/foo.txt +!vendor/.git diff --git a/tests/test-data/projects/no-docker-compose/dockerignore1/.gitignore b/tests/test-data/projects/no-docker-compose/dockerignore1/.gitignore new file mode 100644 index 00000000..4767b2c9 --- /dev/null +++ b/tests/test-data/projects/no-docker-compose/dockerignore1/.gitignore @@ -0,0 +1,2 @@ +a.txt +src/src-a.txt diff --git a/tests/test-data/projects/no-docker-compose/dockerignore1/Dockerfile b/tests/test-data/projects/no-docker-compose/dockerignore1/Dockerfile new file mode 100644 index 00000000..24a79d08 --- /dev/null +++ b/tests/test-data/projects/no-docker-compose/dockerignore1/Dockerfile @@ -0,0 +1 @@ +FROM busybox diff --git a/tests/test-data/projects/no-docker-compose/dockerignore1/a.txt b/tests/test-data/projects/no-docker-compose/dockerignore1/a.txt new file mode 100644 index 00000000..2e65efe2 --- /dev/null +++ b/tests/test-data/projects/no-docker-compose/dockerignore1/a.txt @@ -0,0 +1 @@ +a \ No newline at end of file diff --git a/tests/test-data/projects/no-docker-compose/dockerignore1/b.txt b/tests/test-data/projects/no-docker-compose/dockerignore1/b.txt new file mode 100644 index 00000000..63d8dbd4 --- /dev/null +++ b/tests/test-data/projects/no-docker-compose/dockerignore1/b.txt @@ -0,0 +1 @@ +b \ No newline at end of file diff --git a/tests/test-data/projects/no-docker-compose/dockerignore1/c.txt b/tests/test-data/projects/no-docker-compose/dockerignore1/c.txt new file mode 100644 index 00000000..3410062b --- /dev/null +++ b/tests/test-data/projects/no-docker-compose/dockerignore1/c.txt @@ -0,0 +1 @@ +c \ No newline at end of file diff --git a/tests/test-data/projects/no-docker-compose/dockerignore1/dot.git/bar.txt b/tests/test-data/projects/no-docker-compose/dockerignore1/dot.git/bar.txt new file mode 100644 index 00000000..5716ca59 --- /dev/null +++ b/tests/test-data/projects/no-docker-compose/dockerignore1/dot.git/bar.txt @@ -0,0 +1 @@ +bar diff --git a/tests/test-data/projects/no-docker-compose/dockerignore1/dot.git/foo.txt b/tests/test-data/projects/no-docker-compose/dockerignore1/dot.git/foo.txt new file mode 100644 index 00000000..257cc564 --- /dev/null +++ b/tests/test-data/projects/no-docker-compose/dockerignore1/dot.git/foo.txt @@ -0,0 +1 @@ +foo diff --git a/tests/test-data/projects/no-docker-compose/dockerignore1/src/.balena/balena.yml b/tests/test-data/projects/no-docker-compose/dockerignore1/src/.balena/balena.yml new file mode 100644 index 00000000..dbb4e7a0 --- /dev/null +++ b/tests/test-data/projects/no-docker-compose/dockerignore1/src/.balena/balena.yml @@ -0,0 +1 @@ +"src-balena.yml" \ No newline at end of file diff --git a/tests/test-data/projects/no-docker-compose/dockerignore1/src/.gitignore b/tests/test-data/projects/no-docker-compose/dockerignore1/src/.gitignore new file mode 100644 index 00000000..7257b4ea --- /dev/null +++ b/tests/test-data/projects/no-docker-compose/dockerignore1/src/.gitignore @@ -0,0 +1 @@ +src-c.txt diff --git a/tests/test-data/projects/no-docker-compose/dockerignore1/src/src-a.txt b/tests/test-data/projects/no-docker-compose/dockerignore1/src/src-a.txt new file mode 100644 index 00000000..04db2da6 --- /dev/null +++ b/tests/test-data/projects/no-docker-compose/dockerignore1/src/src-a.txt @@ -0,0 +1 @@ +src-a \ No newline at end of file diff --git a/tests/test-data/projects/no-docker-compose/dockerignore1/src/src-b.txt b/tests/test-data/projects/no-docker-compose/dockerignore1/src/src-b.txt new file mode 100644 index 00000000..7fe8055d --- /dev/null +++ b/tests/test-data/projects/no-docker-compose/dockerignore1/src/src-b.txt @@ -0,0 +1 @@ +src-b \ No newline at end of file diff --git a/tests/test-data/projects/no-docker-compose/dockerignore1/src/src-c.txt b/tests/test-data/projects/no-docker-compose/dockerignore1/src/src-c.txt new file mode 100644 index 00000000..24c911c2 --- /dev/null +++ b/tests/test-data/projects/no-docker-compose/dockerignore1/src/src-c.txt @@ -0,0 +1 @@ +src-c \ No newline at end of file diff --git a/tests/test-data/projects/no-docker-compose/dockerignore1/vendor/dot.git/vendor-git-contents b/tests/test-data/projects/no-docker-compose/dockerignore1/vendor/dot.git/vendor-git-contents new file mode 100644 index 00000000..4a054717 --- /dev/null +++ b/tests/test-data/projects/no-docker-compose/dockerignore1/vendor/dot.git/vendor-git-contents @@ -0,0 +1 @@ +vendor-git-contents diff --git a/tests/test-data/projects/no-docker-compose/dockerignore2/.dockerignore b/tests/test-data/projects/no-docker-compose/dockerignore2/.dockerignore new file mode 100644 index 00000000..dc701109 --- /dev/null +++ b/tests/test-data/projects/no-docker-compose/dockerignore2/.dockerignore @@ -0,0 +1,3 @@ +a.txt +src/src-a.txt +symlink-b.txt diff --git a/tests/test-data/projects/no-docker-compose/dockerignore2/Dockerfile b/tests/test-data/projects/no-docker-compose/dockerignore2/Dockerfile new file mode 100644 index 00000000..24a79d08 --- /dev/null +++ b/tests/test-data/projects/no-docker-compose/dockerignore2/Dockerfile @@ -0,0 +1 @@ +FROM busybox diff --git a/tests/test-data/projects/no-docker-compose/dockerignore2/a.txt b/tests/test-data/projects/no-docker-compose/dockerignore2/a.txt new file mode 100644 index 00000000..2e65efe2 --- /dev/null +++ b/tests/test-data/projects/no-docker-compose/dockerignore2/a.txt @@ -0,0 +1 @@ +a \ No newline at end of file diff --git a/tests/test-data/projects/no-docker-compose/dockerignore2/b.txt b/tests/test-data/projects/no-docker-compose/dockerignore2/b.txt new file mode 100644 index 00000000..63d8dbd4 --- /dev/null +++ b/tests/test-data/projects/no-docker-compose/dockerignore2/b.txt @@ -0,0 +1 @@ +b \ No newline at end of file diff --git a/tests/test-data/projects/no-docker-compose/dockerignore2/src/src-a.txt b/tests/test-data/projects/no-docker-compose/dockerignore2/src/src-a.txt new file mode 100644 index 00000000..04db2da6 --- /dev/null +++ b/tests/test-data/projects/no-docker-compose/dockerignore2/src/src-a.txt @@ -0,0 +1 @@ +src-a \ No newline at end of file diff --git a/tests/test-data/projects/no-docker-compose/dockerignore2/src/src-b.txt b/tests/test-data/projects/no-docker-compose/dockerignore2/src/src-b.txt new file mode 100644 index 00000000..7fe8055d --- /dev/null +++ b/tests/test-data/projects/no-docker-compose/dockerignore2/src/src-b.txt @@ -0,0 +1 @@ +src-b \ No newline at end of file diff --git a/tests/test-data/projects/no-docker-compose/dockerignore2/symlink-a.txt b/tests/test-data/projects/no-docker-compose/dockerignore2/symlink-a.txt new file mode 120000 index 00000000..3efcbf03 --- /dev/null +++ b/tests/test-data/projects/no-docker-compose/dockerignore2/symlink-a.txt @@ -0,0 +1 @@ +src/src-a.txt \ No newline at end of file diff --git a/tests/test-data/projects/no-docker-compose/dockerignore2/symlink-b.txt b/tests/test-data/projects/no-docker-compose/dockerignore2/symlink-b.txt new file mode 120000 index 00000000..64430821 --- /dev/null +++ b/tests/test-data/projects/no-docker-compose/dockerignore2/symlink-b.txt @@ -0,0 +1 @@ +src/src-b.txt \ No newline at end of file diff --git a/tests/test-data/projects/registry-secrets.json b/tests/test-data/projects/registry-secrets.json new file mode 100644 index 00000000..872d4613 --- /dev/null +++ b/tests/test-data/projects/registry-secrets.json @@ -0,0 +1 @@ +{"https://index.docker.io/v1/":{"username":"test","password":"test"}} \ No newline at end of file diff --git a/tests/utils/tarDirectory.spec.ts b/tests/utils/tarDirectory.spec.ts new file mode 100644 index 00000000..e7ed75f3 --- /dev/null +++ b/tests/utils/tarDirectory.spec.ts @@ -0,0 +1,192 @@ +/** + * @license + * Copyright 2020 Balena Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// tslint:disable-next-line:no-var-requires +require('./../config-tests'); // required for side effects + +import { expect } from 'chai'; +import * as _ from 'lodash'; +import * as path from 'path'; +import * as tar from 'tar-stream'; + +import { tarDirectory } from '../../build/utils/compose'; +import { setupDockerignoreTestData } from '../projects'; + +const repoPath = path.normalize(path.join(__dirname, '..', '..')); +const projectsPath = path.join(repoPath, 'tests', 'test-data', 'projects'); + +interface TarFiles { + [name: string]: { + fileSize?: number; + type?: string | null; + }; +} + +const itSkipWindows = process.platform === 'win32' ? it.skip : it; + +describe('compare new and old tarDirectory implementations', async function() { + const extraContent = 'extra'; + const extraEntry: tar.Headers = { + name: 'extra.txt', + size: extraContent.length, + type: 'file', + }; + const preFinalizeCallback = (pack: tar.Pack) => { + pack.entry(extraEntry, extraContent); + }; + + this.beforeAll(async () => { + await setupDockerignoreTestData(); + }); + + this.afterAll(async () => { + await setupDockerignoreTestData({ cleanup: true }); + }); + + it('should produce the expected file list', async function() { + const dockerignoreProjDir = path.join( + projectsPath, + 'no-docker-compose', + 'dockerignore1', + ); + const expectedFiles = { + '.balena/balena.yml': { fileSize: 12, type: 'file' }, + '.dockerignore': { fileSize: 438, type: 'file' }, + '.gitignore': { fileSize: 20, type: 'file' }, + '.git/foo.txt': { fileSize: 4, type: 'file' }, + 'a.txt': { fileSize: 1, type: 'file' }, + 'c.txt': { fileSize: 1, type: 'file' }, + Dockerfile: { fileSize: 13, type: 'file' }, + 'extra.txt': { fileSize: 5, type: 'file' }, + 'src/.balena/balena.yml': { fileSize: 16, type: 'file' }, + 'src/.gitignore': { fileSize: 10, type: 'file' }, + 'src/src-a.txt': { fileSize: 5, type: 'file' }, + 'src/src-c.txt': { fileSize: 5, type: 'file' }, + 'vendor/.git/vendor-git-contents': { fileSize: 20, type: 'file' }, + }; + + const tarPack = await tarDirectory(dockerignoreProjDir, { + preFinalizeCallback, + nogitignore: true, + }); + const fileList = await getTarPackFiles(tarPack); + + expect(fileList).to.deep.equal(expectedFiles); + }); + + it('should produce the expected file list (symbolic links)', async function() { + const projectPath = path.join( + projectsPath, + 'no-docker-compose', + 'dockerignore2', + ); + const expectedFiles = { + '.dockerignore': { fileSize: 34, type: 'file' }, + 'b.txt': { fileSize: 1, type: 'file' }, + Dockerfile: { fileSize: 13, type: 'file' }, + 'src/src-b.txt': { fileSize: 5, type: 'file' }, + 'symlink-a.txt': { fileSize: 5, type: 'file' }, + }; + + const tarPack = await tarDirectory(projectPath, { nogitignore: true }); + const fileList = await getTarPackFiles(tarPack); + + expect(fileList).to.deep.equal(expectedFiles); + }); + + // Skip Windows because the old tarDirectory() implementation (still used when + // '--nogitignore' is not provided) uses the old `zeit/dockerignore` npm package + // that is broken on Windows (reason why we created `@balena/dockerignore`). + itSkipWindows('should produce a compatible tar stream', async function() { + const dockerignoreProjDir = path.join( + projectsPath, + 'no-docker-compose', + 'dockerignore1', + ); + const oldTarPack = await tarDirectory(dockerignoreProjDir, { + preFinalizeCallback, + nogitignore: false, + }); + const oldFileList = await getTarPackFiles(oldTarPack); + + const newTarPack = await tarDirectory(dockerignoreProjDir, { + preFinalizeCallback, + nogitignore: true, + }); + const newFileList = await getTarPackFiles(newTarPack); + + const gitIgnored = ['a.txt', 'src/src-a.txt', 'src/src-c.txt']; + + expect({ + ...newFileList, + ..._.pick(oldFileList, ['.git/bar.txt']), + }).to.deep.equal({ + ...oldFileList, + ..._.pick(newFileList, gitIgnored), + }); + }); + + itSkipWindows( + 'should produce a compatible tar stream (symbolic links)', + async function() { + const dockerignoreProjDir = path.join( + projectsPath, + 'no-docker-compose', + 'dockerignore2', + ); + const oldTarPack = await tarDirectory(dockerignoreProjDir, { + preFinalizeCallback, + nogitignore: false, + }); + const oldFileList = await getTarPackFiles(oldTarPack); + + const newTarPack = await tarDirectory(dockerignoreProjDir, { + preFinalizeCallback, + nogitignore: true, + }); + const newFileList = await getTarPackFiles(newTarPack); + + expect(newFileList).to.deep.equal(oldFileList); + }, + ); +}); + +async function getTarPackFiles( + pack: import('stream').Readable, +): Promise { + const { drainStream } = await import('tar-utils'); + const fileList: TarFiles = {}; + const extract = tar.extract(); + + return await new Promise((resolve, reject) => { + extract + .on('error', reject) + .on('entry', async function(header, stream, next) { + expect(fileList).to.not.have.property(header.name); + fileList[header.name] = { + fileSize: header.size, + type: header.type, + }; + await drainStream(stream); + next(); + }) + .on('finish', function() { + resolve(fileList); + }); + pack.pipe(extract); + }); +}