Add a supervisor endpoint to cleanup orphaned volumes

Change-type: minor
Signed-off-by: Cameron Diver <cameron@balena.io>

docs/API.md

@@ -1152,3 +1152,33 @@ Response:
 	]
 }
 ```
+### V2 Utilities
+
+#### Cleanup volumes with no references
+Added in supervisor version v10.0.0
+
+Starting with balena-supervisor v10.0.0, volumes which have no
+references are no longer automatically removed as part of
+the standard update flow. To cleanup up any orphaned
+volumes, use this supervisor endpoint:
+
+From an application container:
+```
+$ curl "$BALENA_SUPERVISOR_ADDRESS/v2/cleanup-volumes?apikey=$BALENA_SUPERVISOR_API_KEY"
+```
+
+Successful response:
+```
+{
+       "status": "success"
+}
+```
+
+Unsuccessful response:
+```
+{
+       "status": "failed",
+       "message": "the error message"
+}
+```
+

package-lock.json

@@ -1609,17 +1609,8 @@
         "deep-equal": "^1.0.1",
         "dns-equal": "^1.0.0",
         "dns-txt": "^2.0.2",
+        "multicast-dns": "git+https://github.com/resin-io-modules/multicast-dns.git#a15c63464eb43e8925b187ed5cb9de6892e8aacc",
         "multicast-dns-service-types": "^1.1.0"
-      },
-      "dependencies": {
-        "multicast-dns": {
-          "version": "git+https://github.com/resin-io-modules/multicast-dns.git#a15c63464eb43e8925b187ed5cb9de6892e8aacc",
-          "from": "git+https://github.com/resin-io-modules/multicast-dns.git#a15c63464eb43e8925b187ed5cb9de6892e8aacc",
-          "requires": {
-            "dns-packet": "^1.0.1",
-            "thunky": "^0.1.0"
-          }
-        }
       }
     },
     "brace-expansion": {
@@ -2803,6 +2794,16 @@
       "integrity": "sha1-s55/HabrCnW6nBcySzR1PEfgZU0=",
       "dev": true
     },
+    "dns-packet": {
+      "version": "1.3.1",
+      "resolved": "https://registry.npmjs.org/dns-packet/-/dns-packet-1.3.1.tgz",
+      "integrity": "sha512-0UxfQkMhYAUaZI+xrNZOz/as5KgDU0M/fQ9b6SpkyLbk3GEswDi6PADJVaYJradtRVsRIlF1zLyOodbcTCDzUg==",
+      "dev": true,
+      "requires": {
+        "ip": "^1.1.0",
+        "safe-buffer": "^5.0.1"
+      }
+    },
     "dns-txt": {
       "version": "2.0.2",
       "resolved": "https://registry.npmjs.org/dns-txt/-/dns-txt-2.0.2.tgz",
@@ -7307,6 +7308,15 @@
       "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=",
       "dev": true
     },
+    "multicast-dns": {
+      "version": "git+https://github.com/resin-io-modules/multicast-dns.git#a15c63464eb43e8925b187ed5cb9de6892e8aacc",
+      "from": "git+https://github.com/resin-io-modules/multicast-dns.git#listen-on-all-interfaces",
+      "dev": true,
+      "requires": {
+        "dns-packet": "^1.0.1",
+        "thunky": "^0.1.0"
+      }
+    },
     "multicast-dns-service-types": {
       "version": "1.1.0",
       "resolved": "https://registry.npmjs.org/multicast-dns-service-types/-/multicast-dns-service-types-1.1.0.tgz",
@@ -10053,6 +10063,12 @@
         "xtend": "~4.0.1"
       }
     },
+    "thunky": {
+      "version": "0.1.0",
+      "resolved": "https://registry.npmjs.org/thunky/-/thunky-0.1.0.tgz",
+      "integrity": "sha1-vzAUaCTituZ7Dy16Ssi+smkIaE4=",
+      "dev": true
+    },
     "tildify": {
       "version": "1.2.0",
       "resolved": "https://registry.npmjs.org/tildify/-/tildify-1.2.0.tgz",

src/application-manager.d.ts

@@ -14,6 +14,9 @@ import { APIBinder } from './api-binder';
 import { Service } from './compose/service';
 import Config from './config';
 
+import NetworkManager from './compose/network-manager';
+import VolumeManager from './compose/volume-manager';
+
 declare interface Options {
 	force?: boolean;
 	running?: boolean;
@@ -41,6 +44,8 @@ export class ApplicationManager extends EventEmitter {
 	public apiBinder: APIBinder;
 
 	public services: ServiceManager;
+	public volumes: VolumeManager;
+	public networks: NetworkManager;
 	public config: Config;
 	public db: DB;
 	public images: Images;

src/compose/volume-manager.ts

@@ -1,7 +1,5 @@
 import * as Docker from 'dockerode';
-import filter = require('lodash/filter');
-import get = require('lodash/get');
-import unionBy = require('lodash/unionBy');
+import * as _ from 'lodash';
 import * as Path from 'path';
 
 import constants = require('../lib/constants');
@@ -53,7 +51,7 @@ export class VolumeManager {
 
 	public async getAllByAppId(appId: number): Promise<Volume[]> {
 		const all = await this.getAll();
-		return filter(all, { appId });
+		return _.filter(all, { appId });
 	}
 
 	public async create(volume: Volume): Promise<void> {
@@ -131,6 +129,34 @@ export class VolumeManager {
 		return volume;
 	}
 
+	public async removeOrphanedVolumes(): Promise<void> {
+		// Iterate through every container, and track the
+		// references to a volume
+		// Note that we're not just interested in containers
+		// which are part of the private state, and instead
+		// *all* containers. This means we don't remove
+		// something that's part of a sideloaded container
+		const [dockerContainers, dockerVolumes] = await Promise.all([
+			this.docker.listContainers(),
+			this.docker.listVolumes(),
+		]);
+
+		const containerVolumes = _(dockerContainers)
+			.flatMap(c => c.Mounts)
+			.filter(m => m.Type === 'volume')
+			// We know that the name must be set, if the mount is
+			// a volume
+			.map(m => m.Name as string)
+			.uniq()
+			.value();
+		const volumeNames = _.map(dockerVolumes.Volumes, 'Name');
+
+		const volumesToRemove = _.difference(volumeNames, containerVolumes);
+		await Promise.all(
+			volumesToRemove.map(v => this.docker.getVolume(v).remove()),
+		);
+	}
+
 	private async listWithBothLabels(): Promise<Docker.VolumeInspectInfo[]> {
 		const [legacyResponse, currentResponse] = await Promise.all([
 			this.docker.listVolumes({
@@ -141,9 +167,9 @@ export class VolumeManager {
 			}),
 		]);
 
-		const legacyVolumes = get(legacyResponse, 'Volumes', []);
-		const currentVolumes = get(currentResponse, 'Volumes', []);
-		return unionBy(legacyVolumes, currentVolumes, 'Name');
+		const legacyVolumes = _.get(legacyResponse, 'Volumes', []);
+		const currentVolumes = _.get(currentResponse, 'Volumes', []);
+		return _.unionBy(legacyVolumes, currentVolumes, 'Name');
 	}
 }
 

src/device-api/v2.ts

@@ -485,4 +485,18 @@ export function createV2Api(router: Router, applications: ApplicationManager) {
 			});
 		}
 	});
+
+	router.get('/v2/cleanup-volumes', async (_req, res) => {
+		try {
+			await applications.volumes.removeOrphanedVolumes();
+			res.json({
+				status: 'success',
+			});
+		} catch (e) {
+			res.status(503).json({
+				status: 'failed',
+				message: messageFromError(e),
+			});
+		}
+	});
 }