Setup docker-compose.test

This allows to run integration tests during development and on CI
with the right dependencies. There are several changes that this
involves, but the gist of it is that a test environment is setup using
`docker-compose.test.yml`. This file is loaded by `resin-ci` during the
build, and ensures that integration tests are ran after setting up all
requirements. This commit also defines a test environment command that
can be setup using `npm run test:env` in order to run tests in a local
development machine.

.mochapodrc.yml

@@ -1,23 +1,10 @@
-# IP address or URL for the docker host. If no protocol is included, the protocol
-# is assumed to be `tcp://`
-# e.g.
-# - `tcp://192.168.1.105`
-# - `unix:///var/run/docker.sock`
-#
-# It defaults to unix:///var/run/docker.sock
-#
-dockerHost: unix:///var/run/docker.sock
-
-# Use command from Dockerfile when testing
-testCommand: []
-
-# Additional options to pass to the image build
-dockerBuildOpts:
-  target: 'test' # this is the stage in the Dockerfile that runs the test
-
 # This defines default configuration that
-# should apply to all testfs instances
+# should apply to all mocha-pod/testfs instances
 testfs:
+  # These are test files that need to be configured
+  # for all tests. The `from` defines where to find
+  # them in the local source. These can be overriden
+  # in the `testfs` configuration.
   filesystem:
     /mnt/boot/config.json:
       from: test/data/config.json
@@ -27,5 +14,10 @@ testfs:
       from: test/data/mnt/boot/device-type.json
     /etc/os-release:
       from: test/data/etc/os-release
+  # The `keep` list defines files that already exist in the
+  # filesystem and need to be backed up before setting up the test environment
+  keep: []
+  # The `cleanup` list defines files created by the tests that need to be removed
+  # when restoring the filesystem
   cleanup:
     - /data/database.sqlite

.resinci.yml

@@ -1,3 +1,9 @@
 reviewers: 1
-docker: false
+docker:
+  publish: false
+  builds:
+    - dockerfile: Dockerfile.null
+      docker_repo: dev-null
+      path: .
+      publish: false
 npm: false

Dockerfile.null

@@ -0,0 +1,5 @@
+# This dockerfile is used to trick resinCI to pick up 
+# docker-compose.test.yml but without the CI needing to do extra
+# work building a throwaway image. The real build happens in the balena builder
+# so for testing we just want to build up to the test stage
+FROM alpine

Dockerfile.template

@@ -101,6 +101,13 @@ COPY typings ./typings
 COPY src ./src
 COPY test ./test
 
+# Run type checking and unit/legacy tests here 
+# to prevent setting up a test environment that will 
+# most likely fail.
+# For now this also runs the legacy tests, that are slow, this is a 
+# forcing function to finish the split between unit/integration
+RUN npm run test:base
+
 # When running tests from a container built from this stage, 
 # skip the mocha-pod setup
 ENV MOCHAPOD_SKIP_SETUP=1
@@ -109,14 +116,6 @@ ENV MOCHAPOD_SKIP_SETUP=1
 # from this stage
 CMD npm run test:integration
 
-##################################################
-# CI tests are ran here
-##################################################
-FROM test as test-ci
-
-# Run tests
-RUN MOCHAPOD_SKIP_SETUP=1 npm run test
-
 ###################################################
 # Build the production package
 ###################################################

README.md

@@ -176,32 +176,64 @@ balena build -d raspberrypi4-64 -A aarch64
 
 ## Testing
 
-You can run the supervisor test suite with:
+The codebase splits the test suite into unit and integration tests. While unit tests can be run in the local development machine,
+integration tests require a containerized environment with the right dependencies to be setup.
+
+To run type checks, and unit tests, you can use:
 
 ```
-npm test
+npm run test:base
 ```
 
 The supervisor runs on Node v12.16.2, so using that specific
 version will ensure tests run in the same environment as
 production.
 
-Alternatively, tests will be run when building the image,
-which ensures that the environment is exactly the same.
+In order to run all tests, unit and integration, [docker-compose](https://docs.docker.com/compose/) is required.
+The following command will build the image for an `amd64` target (running `test:base` during the build) and setup
+the necessary dependencies for running integration tests.
+
+```
+npm run test:compose
+```
+
+This is the same process that happens in the Supervisor repository CI system, to ensure released supervisor versions have been
+properly tested.
+
+Alternatively, you can launch a test environment using the following commands.
+
+```
+# Launch the environment
+npm run test:env
+
+# In another terminal, access the sut container
+docker-compose exec -i sut sh
+
+# Or alternatively, access the sut container using docker exec
+docker exec -ti $(docker ps -q --filter="name=_sut") sh
+```
+
+And then run all tests using
+
+```
+npm run test
+```
+
+For more information about testing the Supervisor, see the [testing README](test/README.md).
 
 #### Running specific tests quickly
 
-You can run specific tests quickly with the `test:node` script by matching with test suites (describe) or test cases (it) using a string or regexp:
+You can run specific tests quickly with the `test:unit` script or the `test:integration` script (if working in a test environment) by matching with test suites (describe) or test cases (it) using a string or regexp:
 
 ```sh
-npm run test:node -- -g spawnJournalctl
+npm run test:unit -- -g "JSON utils"
 
-npm run test:node -- -g "detect a V2 delta"
+npm run test:unit -- -g "detect a V2 delta"
 
-npm run test:node -- -g (GET|POST|PUT|DELETE)
+npm run test:unit -- -g "(GET|POST|PUT|DELETE)"
 ```
 
-The --grep option, when specified, will trigger mocha to only run tests matching the given pattern which is internally compiled to a RegExp.
+The `--grep` option, when specified, will trigger mocha to only run tests matching the given pattern which is internally compiled to a RegExp.
 
 ## Troubleshooting
 

docker-compose.dev.yml

@@ -1,24 +1,8 @@
-version: '2'
+version: '2.3'
 
 services:
-  docker:
-    image: docker:dind
-    privileged: true
-    environment:
-      DOCKER_TLS_CERTDIR: ''
-    command: --tls=false --debug
-
   sut:
-    build:
-      context: ./
-      dockerfile: Dockerfile.template
-      target: test
-      args:
-        # Change this if testing in another architecture
-        ARCH: amd64
     command: sleep infinity
-    environment:
-      DOCKER_HOST: tcp://docker:2375
     volumes:
       - './.mochapodrc.yml:/usr/src/app/.mochapodrc.yml'
       - './src:/usr/src/app/src'

docker-compose.test.yml

@@ -0,0 +1,40 @@
+version: '2.3'
+
+services:
+  # Override the supervisor service configuration. The tests will
+  # be run through the `sut` service
+  balena-supervisor:
+    build:
+      dockerfile: Dockerfile.null
+      context: ./
+    command: sleep infinity
+
+  docker:
+    image: docker:dind
+    privileged: true
+    environment:
+      DOCKER_TLS_CERTDIR: ''
+    command: --tls=false --debug
+
+  sut:
+    # Build the supervisor code for development and testing
+    build:
+      context: ./
+      dockerfile: Dockerfile.template
+      target: test
+      args:
+        # Change this if testing in another architecture
+        ARCH: amd64
+    depends_on:
+      - balena-supervisor
+      - docker
+    # Set required supervisor configuration variables here
+    environment:
+      DOCKER_HOST: tcp://docker:2375
+    # Set required mounts as tmpfs or volumes here
+    # if specific files need to be backed up between tests,
+    # make sure to add them to the `testfs` configuration under
+    # .mochapodrc.yml
+    tmpfs:
+      - /data
+      - /mnt/boot

docker-compose.yml

@@ -1,4 +1,4 @@
-version: '2.1'
+version: '2.3'
 
 services:
   # This service can only be named `main` or `balena-supervisor` as older

package.json

@@ -16,9 +16,11 @@
 		"test:integration": "mocha --config test/integration/.mocharc.js",
 		"test:legacy": "mocha --config test/legacy/.mocharc.js",
 		"test:node": "npm run test:unit && npm run test:integration && npm run test:legacy",
-		"test:env": "docker-compose -f docker-compose.dev.yml up --build || npm run compose:down",
-		"test": "npm run lint && npm run test:build  && npm run test:node",
-		"compose:down": "docker-compose -f docker-compose.dev.yml down",
+		"test:env": "docker-compose -f docker-compose.test.yml -f docker-compose.dev.yml up --build; npm run compose:down",
+		"test:compose": "docker-compose -f docker-compose.yml -f docker-compose.test.yml up --build --remove-orphans --exit-code-from=sut ; npm run compose:down",
+		"test:base": "npm run lint && npm run test:build && npm run test:unit && npm run test:legacy",
+		"test": "npm run lint && npm run test:build && npm run test:node",
+		"compose:down": "docker-compose -f docker-compose.test.yml down",
 		"prettify": "balena-lint -e ts -e js --typescript --fix src/ test/ typings/ build-utils/ webpack.config.js",
 		"release": "tsc --project tsconfig.release.json && mv build/src/* build",
 		"sync": "ts-node --files sync/sync.ts",

test/integration/.mocharc.js

@@ -7,7 +7,7 @@ module.exports = {
 		'ts-node/register/transpile-only',
 		'tsconfig-paths/register',
 		'test/lib/chai.ts',
-		'mocha-pod/attach',
+		'mocha-pod/skip-setup',
 	],
 	spec: ['test/integration/**/*.spec.ts'],
 	timeout: '30000',