Respect ignore files when tarring sources

This commit brings in the ignore and dockerignore libraries, which when
provided with the patterns in the aforementioned files will ignore them.

Change-type: major
Closes: 889
Signed-off-by: Cameron Diver <cameron@resin.io>

lib/utils/compose.coffee

@@ -108,14 +108,21 @@ exports.tarDirectory = tarDirectory = (dir) ->
 	path = require('path')
 	fs = require('mz/fs')
 	streamToPromise = require('stream-to-promise')
+	{ FileIgnorer } = require('./ignore')
 
 	getFiles = ->
 		streamToPromise(klaw(dir))
 		.filter((item) -> not item.stats.isDirectory())
 		.map((item) -> item.path)
 
+	ignore = new FileIgnorer(dir)
 	pack = tar.pack()
 	getFiles(dir)
+	.each (file) ->
+		type = ignore.getIgnoreFileType(path.relative(dir, file))
+		if type?
+			ignore.addIgnoreFile(file, type)
+	.filter(ignore.filter)
 	.map (file) ->
 		relPath = path.relative(path.resolve(dir), file)
 		Promise.join relPath, fs.stat(file), fs.readFile(file),

lib/utils/ignore.ts

@@ -0,0 +1,149 @@
+import * as _ from 'lodash';
+import { fs } from 'mz';
+import * as path from 'path';
+
+import dockerIgnore = require('@zeit/dockerignore');
+import ignore from 'ignore';
+
+export enum IgnoreFileType {
+	DockerIgnore,
+	GitIgnore,
+}
+
+interface IgnoreEntry {
+	pattern: string;
+	// The relative file path from the base path of the build context
+	filePath: string;
+}
+
+export class FileIgnorer {
+	private dockerIgnoreEntries: IgnoreEntry[];
+	private gitIgnoreEntries: IgnoreEntry[];
+
+	private static ignoreFiles: Array<{
+		pattern: string;
+		type: IgnoreFileType;
+		allowSubdirs: boolean;
+	}> = [
+		{
+			pattern: '.gitignore',
+			type: IgnoreFileType.GitIgnore,
+			allowSubdirs: true,
+		},
+		{
+			pattern: '.dockerignore',
+			type: IgnoreFileType.DockerIgnore,
+			allowSubdirs: false,
+		},
+	];
+
+	public constructor(public basePath: string) {
+		this.dockerIgnoreEntries = [];
+		this.gitIgnoreEntries = [];
+	}
+	/**
+	 * @param  {string} relativePath
+	 * 	The relative pathname from the build context, for example a root level .gitignore should be
+	 * 		./.gitignore
+	 * @returns IgnoreFileType
+	 * 	The type of ignore file, or null
+	 */
+	public getIgnoreFileType(relativePath: string): IgnoreFileType | null {
+		for (const { pattern, type, allowSubdirs } of FileIgnorer.ignoreFiles) {
+			if (
+				path.basename(relativePath) === pattern &&
+				(allowSubdirs || path.dirname(relativePath) === '.')
+			) {
+				return type;
+			}
+		}
+
+		return null;
+	}
+	/**
+	 * @param  {string} fullPath
+	 * 	The full path on disk of the ignore file
+	 * @param  {IgnoreFileType} type
+	 * @returns Promise
+	 */
+	public async addIgnoreFile(
+		fullPath: string,
+		type: IgnoreFileType,
+	): Promise<void> {
+		const contents = await fs.readFile(fullPath, 'utf8');
+
+		contents.split('\n').forEach(line => {
+			// ignore empty lines and comments
+			if (/\s*#/.test(line) || _.isEmpty(line)) {
+				return;
+			}
+			this.addEntry(line, fullPath, type);
+		});
+
+		return;
+	}
+
+	// Pass this function as a predicate to a filter function, and it will filter
+	// any ignored files
+	public filter = (filename: string): boolean => {
+		const dockerIgnoreHandle = dockerIgnore();
+		const gitIgnoreHandle = ignore();
+
+		interface IgnoreHandle {
+			add: (pattern: string) => void;
+			ignores: (file: string) => boolean;
+		}
+
+		const ignoreTypes: Array<{
+			handle: IgnoreHandle;
+			entries: IgnoreEntry[];
+		}> = [
+			{ handle: dockerIgnoreHandle, entries: this.dockerIgnoreEntries },
+			{ handle: gitIgnoreHandle, entries: this.gitIgnoreEntries },
+		];
+
+		const relFile = path.relative(this.basePath, filename);
+
+		_.each(ignoreTypes, ({ handle, entries }) => {
+			_.each(entries, ({ pattern, filePath }) => {
+				if (FileIgnorer.contains(path.posix.dirname(filePath), filename)) {
+					handle.add(pattern);
+				}
+			});
+		});
+
+		return !_.some(ignoreTypes, ({ handle }) => handle.ignores(relFile));
+	};
+
+	private addEntry(
+		pattern: string,
+		filePath: string,
+		type: IgnoreFileType,
+	): void {
+		const entry: IgnoreEntry = { pattern, filePath };
+		switch (type) {
+			case IgnoreFileType.DockerIgnore:
+				this.dockerIgnoreEntries.push(entry);
+				break;
+			case IgnoreFileType.GitIgnore:
+				this.gitIgnoreEntries.push(entry);
+				break;
+		}
+	}
+
+	/**
+	 * Given two paths, check whether the first contains the second
+	 * @param path1 The potentially containing path
+	 * @param path2 The potentially contained path
+	 * @return A boolean indicating whether `path1` contains `path2`
+	 */
+	private static contains(path1: string, path2: string): boolean {
+		// First normalise the input, to remove any path weirdness
+		path1 = path.posix.normalize(path1);
+		path2 = path.posix.normalize(path2);
+
+		// Now test if the start of the relative path contains ../ ,
+		// which would tell us that path1 is not part of path2
+		return !/^\.\.\//.test(path.posix.relative(path1, path2));
+	}
+}

package.json

@@ -92,6 +92,7 @@
     "@resin.io/valid-email": "^0.1.0",
     "@types/stream-to-promise": "2.2.0",
     "@types/through2": "^2.0.33",
+    "@zeit/dockerignore": "0.0.1",
     "JSONStream": "^1.0.3",
     "ansi-escapes": "^2.0.0",
     "any-promise": "^1.3.0",
@@ -119,11 +120,13 @@
     "global-tunnel-ng": "^2.1.1",
     "hasbin": "^1.2.3",
     "humanize": "0.0.9",
+    "ignore": "^5.0.2",
     "inquirer": "^3.1.1",
     "is-root": "^1.0.0",
     "js-yaml": "^3.10.0",
     "klaw": "^3.0.0",
     "lodash": "^4.17.4",
+    "minimatch": "^3.0.4",
     "mixpanel": "^0.4.0",
     "mkdirp": "^0.5.1",
     "moment": "^2.20.1",