v4.1.20

Update dependency balena-io/balena-cli to v18.2.20

.github/actionlint.yaml

@@ -0,0 +1,8 @@
+self-hosted-runner:
+  # Labels of self-hosted runner in array of strings.
+  labels:
+  - "distro:jammy"
+# Configuration variables in array of strings defined in your repository or
+# organization. `null` means disabling configuration variables check.
+# Empty array means no configuration variable is allowed.
+config-variables: null

.github/workflows/flowzone.yml

@@ -0,0 +1,52 @@
+name: Flowzone
+
+on:
+  pull_request:
+    types: [opened, synchronize, closed]
+    branches: [main, master]
+  pull_request_target:
+    types: [opened, synchronize, closed]
+    branches: [main, master]
+
+jobs:
+  flowzone:
+    name: Flowzone
+    uses: product-os/flowzone/.github/workflows/flowzone.yml@master
+    # prevent duplicate workflow executions for pull_request and pull_request_target
+    if: |
+      (
+        github.event.pull_request.head.repo.full_name == github.repository &&
+        github.event_name == 'pull_request'
+      ) || (
+        github.event.pull_request.head.repo.full_name != github.repository &&
+        github.event_name == 'pull_request_target'
+      )
+    secrets: inherit
+    with:
+      jobs_timeout_minutes: 60
+      cloudflare_website: open-balena
+      custom_runs_on: |
+        [
+          [
+            "self-hosted",
+            "Linux",
+            "X64"
+          ]
+        ]
+
+      balena_slugs: |
+        balena/open-balena
+
+  tests:
+    name: openBalena tests
+    uses: ./.github/workflows/tests.yml
+    needs: [flowzone]
+    if: |
+      ((
+        github.event.pull_request.head.repo.full_name == github.repository &&
+        github.event_name == 'pull_request'
+      ) || (
+        github.event.pull_request.head.repo.full_name != github.repository &&
+        github.event_name == 'pull_request_target'
+      )) && github.event.action != 'closed'
+    secrets: inherit

.github/workflows/renovate.json

@@ -0,0 +1,15 @@
+{
+  "customManagers": [
+    {
+      "customType": "regex",
+      "fileMatch": [
+        ".*"
+      ],
+      "matchStrings": [
+        ".*amiFilter=(?<packageName>.*?)\n(.*currentImageName=(?<currentDigest>.*?)\n)?(.*\n)?.*?(?<depName>[a-zA-Z0-9-_:]*)[ ]*?[:|=][ ]*?[\"|']?(?<currentValue>ami-[a-z0-9]{17})[\"|']?.*"
+      ],
+      "datasourceTemplate": "aws-machine-image",
+      "versioningTemplate": "aws-machine-image"
+    }
+  ]
+}

.gitignore

@@ -1,7 +1,3 @@
 .DS_Store
-.project
+.balena
-.vagrant/
+**/.env
-
-/config
-/docker-compose.yml
-/package-lock.json

.openbalenarc

@@ -1,50 +0,0 @@
-#!/bin/bash
-
-alias dc="/home/vagrant/openbalena/scripts/compose"
-
-function enter () {
-	if [[ $# -lt 1 ]]; then			
-		echo "Usage: enter <service name> [command]"
-		echo "  "
-		echo "  Runs a [command] in the service specified."
-		echo "  "
-		echo "  command:"
-		echo "    (default) /bin/bash"
-		echo "  "
-		echo "  example:"
-		echo "    enter api          # this will run the command '/bin/bash' in the API service, providing a shell prompt"
-		echo "    enter api uptime   # this will run the command 'uptime' in the API service, and return"
-		return 1
-	fi
-
-
-	service="$1"
-	shift
-	COMMAND=/bin/bash
-	if [[ $# -gt 0 ]]; then
-		COMMAND="$@"
-	fi
-	dc exec ${service} /bin/bash -c "${COMMAND}"
-}
-
-function logs () {
-	if [[ $# -lt 1 ]]; then			
-		echo "Usage: logs <service name> [options]"
-		echo "  "
-		echo "  Shows the logs from journalctl in the service specified."
-		echo "  "
-		echo "  options:"
-		echo "    -f  tail the log stream"
-		echo "    -n  number of lines to take"
-		echo "  "
-		echo "  example:"
-		echo "    logs api -fn100 # this will tail the API log, starting with the last 100 lines"
-		return 1
-	fi
-
-	service="$1"
-	shift
-	enter ${service} journalctl "$@"
-}
-
-cd /home/vagrant/openbalena

Makefile

@@ -1,4 +1,159 @@
-.PHONY: lint
+SHELL := bash
 
-lint:
+# export all variables to child processes by default
-	shellcheck scripts/*
+export
+
+# include the .env file if it exists
+-include .env
+
+BALENARC_NO_ANALYTICS ?= 1
+DNS_TLD ?= $(error DNS_TLD not set)
+ORG_UNIT ?= openBalena
+PRODUCTION_MODE ?= true
+STAGING_PKI ?= /usr/local/share/ca-certificates
+SUPERUSER_EMAIL ?= admin@$(DNS_TLD)
+TMPKI := $(shell mktemp)
+VERBOSE ?= false
+
+.NOTPARALLEL: $(DOCKERCOMPOSE)
+
+.PHONY: help
+help: ## Print help message
+	@echo -e "$$(grep -hE '^\S+:.*##' $(MAKEFILE_LIST) | sed -e 's/:.*##\s*/:/' -e 's/^\(.\+\):\(.*\)/\\x1b[36m\1\\x1b[m:\2/' | column -c2 -t -s :)"
+
+.PHONY: lint
+lint: ## Lint shell scripts with shellcheck
+	find . -type f -name *.sh | xargs shellcheck
+
+.PHONY: verify
+verify: ## Ping the public API endpoint
+	curl --fail --retry 3 https://api.$(DNS_TLD)/ping
+	@printf '\n'
+
+# Write all supported variables to .env, whether they have been provided or not.
+# If they already exist in the .env they will be retained.
+# The existing .env takes priority over envs provided from the command line.
+.PHONY: config
+config: ## Rewrite the .env config from current context (env vars + env args + existing .env)
+ifneq ($(CLOUDFLARE_API_TOKEN),)
+ifneq ($(GANDI_API_TOKEN),)
+	$(error "CLOUDFLARE_API_TOKEN and GANDI_API_TOKEN cannot both be set")
+endif
+endif
+	@rm -f .env
+	@echo "BALENARC_NO_ANALYTICS=$(BALENARC_NO_ANALYTICS)" > .env
+	@echo "DNS_TLD=$(DNS_TLD)" >> .env
+	@echo "ORG_UNIT=$(ORG_UNIT)" >> .env
+	@echo "PRODUCTION_MODE=$(PRODUCTION_MODE)" >> .env
+	@echo "SUPERUSER_EMAIL=$(SUPERUSER_EMAIL)" >> .env
+	@echo "VERBOSE=$(VERBOSE)" >> .env
+ifneq ($(ACME_EMAIL),)
+	@echo "ACME_EMAIL=$(ACME_EMAIL)" >> .env
+endif
+ifneq ($(CLOUDFLARE_API_TOKEN),)
+	@echo "CLOUDFLARE_API_TOKEN=$(CLOUDFLARE_API_TOKEN)" >> .env
+endif
+ifneq ($(GANDI_API_TOKEN),)
+	@echo "GANDI_API_TOKEN=$(GANDI_API_TOKEN)" >> .env
+endif
+ifneq ($(HAPROXY_CRT),)
+	@echo "HAPROXY_CRT=$(HAPROXY_CRT)" >> .env
+endif
+ifneq ($(HAPROXY_KEY),)
+	@echo "HAPROXY_KEY=$(HAPROXY_KEY)" >> .env
+endif
+ifneq ($(ROOT_CA),)
+	@echo "ROOT_CA=$(ROOT_CA)" >> .env
+endif
+	@$(MAKE) showenv
+
+.PHONY: wait
+wait: ## Wait for service
+	@until [[ $$(docker compose ps $(SERVICE) --format json | jq -r '.Health') =~ ^healthy$$ ]]; do printf '.'; sleep 3; done
+	@printf '\n'
+
+.PHONY: waitlog
+waitlog: ## Wait for log line
+	@until docker compose logs $(SERVICE) | grep -Eq "$(LOG_STRING)"; do printf '.'; sleep 3; done
+
+.PHONY: up
+up: config ## Start all services
+	@docker compose up --build -d
+	@$(MAKE) wait SERVICE=api
+	@$(MAKE) showenv
+	@$(MAKE) showpass
+
+.PHONY: showenv
+showenv: ## Print the current contents of the .env config
+	@cat <.env
+	@printf '\n'
+
+.PHONY: printenv
+printenv: ## Print the current environment variables
+	@printenv
+
+.PHONY: showpass
+showpass: ## Print the superuser password
+	@docker compose exec api cat config/env | grep SUPERUSER_PASSWORD
+	@printf '\n'
+
+.PHONY: down
+down: ## Stop all services
+	@docker compose stop
+
+.PHONY: stop
+stop: down ## Alias for 'make down'
+
+.PHONY: restart
+restart: ## Restart all services
+	@docker compose restart
+	@$(MAKE) wait SERVICE=api
+
+.PHONY: update
+update: # Pull and deploy latest changes from git
+	@git pull
+	@$(MAKE) up
+
+.PHONY: destroy ## Stop and remove any existing containers and volumes
+destroy:
+	@docker compose down --volumes --remove-orphans
+
+.PHONY: clean
+clean: destroy ## Alias for 'make destroy'
+
+.PHONY: self-signed
+self-signed: ## Install self-signed CA certificates
+	@sudo mkdir -p .balena $(STAGING_PKI)
+
+	@true | openssl s_client -showcerts -connect api.$(DNS_TLD):443 \
+	  | awk '/BEGIN CERTIFICATE/,/END CERTIFICATE/ {print $0}' > $(TMPKI).ca
+
+	@cat <$(TMPKI).ca | openssl x509 -text \
+	  | awk '/BEGIN CERTIFICATE/,/END CERTIFICATE/ {print $0}' > $(TMPKI).srv
+
+	@diff --suppress-common-lines --unchanged-line-format= \
+	  $(TMPKI).srv \
+	  $(TMPKI).ca | sudo tee $(STAGING_PKI)/ca-$(DNS_TLD).crt || true
+
+	@sudo update-ca-certificates
+	@cat <$(STAGING_PKI)/ca-$(DNS_TLD).crt | sudo tee .balena/ca-$(DNS_TLD).pem
+
+# FIXME: refactor this function to use 'make up'
+.PHONY: auto-pki
+auto-pki: config # Start all services using LetsEncrypt and ACME
+	@docker compose exec cert-manager rm -f /certs/export/chain.pem
+	@docker compose up -d
+	@$(MAKE) waitlog SERVICE=cert-manager LOG_STRING="/certs/export/chain.pem Certificate will not expire in [0-9] days"
+	@$(MAKE) waitlog SERVICE=cert-manager LOG_STRING="subject=CN = ${DNS_TLD}"
+	@$(MAKE) waitlog SERVICE=cert-manager LOG_STRING="issuer=C = US, O = Let's Encrypt, CN = R3"
+	@$(MAKE) wait SERVICE=haproxy
+	@$(MAKE) showenv
+	@$(MAKE) showpass
+
+.PHONY: pki-custom
+pki-custom: up ## Alias for 'make up'
+
+.PHONY: deploy
+deploy: up ## Alias for 'make up'
+
+.DEFAULT_GOAL = help

README.md

@@ -1,6 +1,8 @@
-<img alt="openBalena" src="docs/assets/openbalena-logo.svg" height="82">
+[![Flowzone](https://github.com/balena-io/open-balena/actions/workflows/flowzone.yml/badge.svg)](https://github.com/balena-io/open-balena/actions/workflows/flowzone.yml)
 
----
+![](./docs/images/openbalena-logo.svg)
+
+[![deploy button](https://balena.io/deploy.svg)](https://dashboard.balena-cloud.com/deploy?repoUrl=https://github.com/balena-io/open-balena)
 
 OpenBalena is a platform to deploy and manage connected devices. Devices run
 [balenaOS][balena-os-website], a host operating system designed for running
@@ -36,13 +38,17 @@ application to your device(s).
 
 The current release of openBalena has the following minimum version requirements:
 
-- balenaOS v2.58.3
+- balenaOS v5.2.8
-- balena CLI v12.38.5
+- balena CLI v18.2.2
 
 If you are updating from previous openBalena versions, ensure you update the balena
-CLI and reprovision any devices to at least the minimum required versions in order
+CLI and re-provision any devices to at least the minimum required versions in order
 for them to be fully compatible with this release, as some features may not work.
 
+While in-place openBalena upgrades may succeed, when performing major updates, it is
+recommended for a new instance to be deployed in parallel with the existing one, followed
+by copying state across and pointing a test device to the new instance.
+
 
 ## Documentation
 
@@ -103,26 +109,66 @@ improvements and new functionality is planned:
 
 ## Differences between openBalena and balenaCloud
 
-| openBalena                                          | balenaCloud                                                                                                                                                                                                                                                             |
+Whilst openBalena and balenaCloud share the same core technology, there are some key
-| -----                                               | ----                                                                                                                                                                                                                                                                    |
+differences. First, openBalena is self-hosted, whereas balenaCloud is hosted by balena and
-| Device updates using full images                    | Device updates using [delta images](https://www.balena.io/docs/learn/deploy/delta/)                                                                                                                                                                                     |
+therefore handles security, maintenance, scaling, and reliability of all the backend
-| Support for a single user                           | Support for [multiple users](https://www.balena.io/docs/learn/manage/account/#application-members)                                                                                                                                                                      |
+services. OpenBalena is also single user, whereas balenaCloud supports multiple users and
-| Self-hosted deployment and scaling                  | balena-managed scaling and deployment |
+organizations. OpenBalena also lacks some of the commercial features that define
-| Community support via [forums][forums]              | Private support on [paid plans](https://www.balena.io/pricing/)                                                                                                                                           |
+balenaCloud, such as the web-based dashboard and updates with binary container deltas.
-| Deploy via `balena deploy` only                     | Build remotely with native builders using [`balena push`](https://www.balena.io/docs/learn/deploy/deployment/#balena-push) or  [`git push`](https://www.balena.io/docs/learn/deploy/deployment/#git-push) |
-| No support for building via `git push`              | Use the same CI workflow with [`git push`](https://www.balena.io/docs/learn/deploy/deployment/#git-push)                                                                                                  |
-| No public URL support                               | Serve websites directly from device with [public device URLs](https://www.balena.io/docs/learn/manage/actions/#enable-public-device-url)                                                                  |
-| Management via `balena-cli` only                    | Cloud-based device management dashboard                                                                                                                                                                   |
-| Download images from [balena.io][balena-os-website] | Download preconfigured images directly from the dashboard                                                                                                                                                 |
-| No supported remote diagnostics                     | Remote device diagnostics                                                                                                                                                                                 |
-| Supported devices: Raspberry Pi family, the Intel NUC, the NVIDIA Jetson TX2, and the balenaFin | All the devices listed in balena's [reference documentation](https://www.balena.io/docs/reference/hardware/devices/)   |
 
-Additionally, refer back to the [roadmap](#roadmap) above for planned but not yet implemented features.
+The following table contains the main differences between both:
+
+| openBalena                                                                                 | balenaCloud                                                                                                                                                                                               |
+| ------------------------------------------------------------------------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| Device updates using full Docker images                                                    | Device updates using [delta images](https://www.balena.io/docs/learn/deploy/delta/)                                                                                                                       |
+| Support for a single user                                                                  | Support for [multiple users](https://www.balena.io/docs/learn/manage/account/#application-members)                                                                                                        |
+| Self-hosted deployment and scaling                                                         | balena-managed scaling and deployment                                                                                                                                                                     |
+| Community support via [forums][forums]                                                     | Private support on [paid plans](https://www.balena.io/pricing/)                                                                                                                                           |
+| Build locally and deploy via `balena-cli`                                                  | Build remotely with native builders using [`balena push`](https://www.balena.io/docs/learn/deploy/deployment/#balena-push) or  [`git push`](https://www.balena.io/docs/learn/deploy/deployment/#git-push) |
+| No public device URL support                                                               | Serve websites directly from device with [public device URLs](https://www.balena.io/docs/learn/manage/actions/#enable-public-device-url)                                                                  |
+| Management via `balena-cli` only                                                           | Cloud-based device management dashboard                                                                                                                                                                   |
+| Download images from [balena.io][balena-os-website] and configure locally via `balena-cli` | Download configured images directly from the dashboard                                                                                                                                                    |
+| No remote device diagnostics                                                               | Remote device diagnostics                                                                                                                                                                                 |
+
+Additionally, refer back to the [roadmap](#roadmap) above for planned but not yet
+implemented features.
 
 
 ## License
 
-OpenBalena is licensed under the terms of AGPL v3. See [LICENSE](LICENSE) for details.
+OpenBalena is licensed under the terms of AGPL v3. See [LICENSE] for details.
+
+
+## FAQ
+
+### How do you ensure continuity of openBalena? Are there security patches on openBalena?
+openBalena is an open source initiative which is mostly driven by us, but it also gets
+contributions from the community. We work to keep openBalena as up to date as our
+bandwidth allows, especially with security patches. That said, we do not have a policy or
+guarantee of a software release schedule. However, it is in our best interest to keep
+openBalena updated and patched since we also use it for balenaCloud.
+
+### How do you ensure the "Join" command actually works between openBalena and
+balenaCloud?
+The `balena join ..` command is frequently used for moving devices between openBalena,
+and balenaCloud environments. This command extends `balena os configure ..`, which is the
+basic tool balena uses for configuring devices.
+
+### Is it "production ready"?
+While we actually have some rather large fleets using openBalena, we consider it to be
+perpetually in "beta". This means potentially introducing breaking changes between
+releases.
+
+### Can new device type be added to openBalena?
+openBalena imports the following public [device-types] "out of the box". You can specify
+your own contracts repository by overriding `CONTRACTS_PUBLIC_REPO_NAME`,
+`CONTRACTS_PUBLIC_REPO_OWNER` and `IMAGE_STORAGE_BUCKET` environment variables on the API
+service/container.
+
+### Are there open-source UI dashboards from the community for openBalena?
+Yes! Here are a few:
+- [open-balena-admin / open-balena-ui](https://github.com/dcaputo-harmoni/open-balena-admin) by [dcaputo-harmoni](https://github.com/dcaputo-harmoni) who first posted about [here](https://forums.balena.io/t/open-balena-admin-an-admin-interface-for-openbalena/355324) in our Forums :)
+- [open-balena-dashboard](https://github.com/Razikus/open-balena-dashboard) by [Razikus](https://github.com/Razikus)
 
 
 [balena-cli]: https://github.com/balena-io/balena-cli
@@ -135,10 +181,14 @@ OpenBalena is licensed under the terms of AGPL v3. See [LICENSE](LICENSE) for de
 [forums]: https://forums.balena.io/c/open-balena
 [getting-started]: https://balena.io/open/docs/getting-started
 [issue-tracker]: https://github.com/balena-io/open-balena/issues
+[LICENSE]: https://github.com/balena-io/open-balena/blob/master/LICENSE
+[open-balena-admin / open-balena-ui]: https://github.com/dcaputo-harmoni/open-balena-admin
 [open-balena-api]: https://github.com/balena-io/open-balena-api
+[open-balena-dashboard]: https://github.com/Razikus/open-balena-dashboard
 [open-balena-db]: https://github.com/balena-io/open-balena-db
 [open-balena-registry]: https://github.com/balena-io/open-balena-registry
 [open-balena-s3]: https://github.com/balena-io/open-balena-s3
 [open-balena-vpn]: https://github.com/balena-io/open-balena-vpn
 [open-balena-website]: https://balena.io/open
 [pulls]: https://github.com/balena-io/open-balena/pulls
+[device-types]: https://github.com/balena-io/contracts/blob/master/contracts/hw.device-type

VERSION

@@ -1 +1 @@
-3.4.3
+4.1.20

Vagrantfile

@@ -1,41 +0,0 @@
-Vagrant.require_version '>= 2.2.0'
-
-Vagrant.configure('2') do |config|
-  config.vagrant.plugins = [
-    'vagrant-vbguest',
-    'vagrant-docker-compose'
-  ]
-
-  config.vm.define 'openbalena'
-  config.vm.hostname = 'openbalena-vagrant'
-  config.vm.box = 'bento/ubuntu-18.04'
-
-  config.vm.network "public_network",
-    use_dhcp_assigned_default_route: true
-
-  config.vm.synced_folder '.', '/vagrant', disabled: true
-  config.vm.synced_folder '.', '/home/vagrant/openbalena'
-
-  config.ssh.forward_agent = true
-
-  config.vm.provision :docker
-
-  $provision = <<-SCRIPT
-    DOCKER_COMPOSE_VERSION=1.24.0
-
-    touch /home/vagrant/.bashrc
-    grep -Fxq 'source /home/vagrant/openbalena/.openbalenarc' /home/vagrant/.bashrc || echo 'source /home/vagrant/openbalena/.openbalenarc' >> /home/vagrant/.bashrc
-
-    curl -o- https://raw.githubusercontent.com/creationix/nvm/v0.34.0/install.sh | bash
-    source "/home/vagrant/.nvm/nvm.sh" # This loads nvm
-    nvm install 10.15.0 && nvm use 10.15.0
-
-    # Install a newer version of docker-compose
-    (cd /usr/local/bin; \
-    sudo curl -o docker-compose --silent --location https://github.com/docker/compose/releases/download/$DOCKER_COMPOSE_VERSION/docker-compose-Linux-x86_64; \
-    sudo chmod a+x docker-compose)
-  SCRIPT
-
-  config.vm.provision :shell, privileged: false, inline: $provision
-
-end

balena.yml

@@ -0,0 +1,26 @@
+name: openBalena
+type: sw.application
+description: https://www.balena.io/open
+post-provisioning: |
+  [![Flowzone](https://github.com/balena-io/open-balena/actions/workflows/flowzone.yml/badge.svg)](https://github.com/balena-io/open-balena/actions/workflows/flowzone.yml)
+
+  ## Getting Started
+
+  * https://open-balena.pages.dev/#getting-started
+
+assets:
+  repository:
+    type: blob.asset
+    data:
+      url: 'https://github.com/balena-io/open-balena'
+  logo:
+    type: blob.asset
+    data:
+      url: 'https://raw.githubusercontent.com/balena-io/open-balena/master/logo.png'
+data:
+  defaultDeviceType: generic-amd64
+  supportedDeviceTypes:
+    - generic-amd64
+    - genericx86-64-ext
+    - intel-nuc
+version: 4.1.20

compose/common.yml

@@ -1,17 +0,0 @@
-version: "2.0"
-
-services:
-  component:
-    cap_add:
-      - SYS_ADMIN
-      - SYS_RESOURCE
-    environment:
-      - CONFD_BACKEND=ENV
-    tmpfs:
-      - /run
-      - /sys/fs/cgroup
-    privileged: true
-
-  system:
-    security_opt:
-      - seccomp:unconfined

compose/mdns.yml

@@ -1,31 +0,0 @@
-version: "2.0"
-
-services:
-  balena-mdns-publisher:
-    image: balena/balena-mdns-publisher:${OPENBALENA_MDNS_PUBLISHER_VERSION_TAG}
-    network_mode: "host"
-    cap_add:
-        - SYS_RESOURCE
-        - SYS_ADMIN
-    security_opt:
-        - apparmor:unconfined
-    tmpfs:
-        - /run
-        - /sys/fs/cgroup
-    # balenaOS - Required for host DBus comms. Not required for standalone Linux
-    labels:
-        io.balena.features.dbus: '1'
-        io.balena.features.supervisor-api: '1'
-    environment:
-        CONFD_BACKEND: ENV
-        # The name of the TLD to use. This *must* match certificates used for the rest of
-        # the resin backend (eg. that for BALENA_ROOT_CA if present).
-        MDNS_TLD: ${OPENBALENA_HOST_NAME}
-        # List of subdomains to advertise. This must include all required hosts.
-        MDNS_SUBDOMAINS: '["api", "db", "registry", "s3", "tunnel", "vpn"]'
-        # The expectation is the DBus socket to use is always at the following location.
-        DBUS_SESSION_BUS_ADDRESS: "unix:path=/host/run/dbus/system_bus_socket"
-        # Selects the interface used for incoming connections from the wider subnet.
-        # For NUCs, this is `eno1`. If running natively, pick the appropriate interface.
-        # Alternatively, keep the default commented out to autoselect.
-        #INTERFACE: "eno1"

compose/services.yml

@@ -1,186 +0,0 @@
-version: "2.0"
-
-volumes:
-  certs: {}
-  cert-provider: {}
-  db: {}
-  redis: {}
-  s3: {}
-
-services:
-  api:
-    extends:
-      file: ./common.yml
-      service: component
-    image: balena/open-balena-api:${OPENBALENA_API_VERSION_TAG}
-    depends_on:
-      - db
-      - s3
-      - redis
-    environment:
-      API_VPN_SERVICE_API_KEY: ${OPENBALENA_API_VPN_SERVICE_API_KEY}
-      BALENA_ROOT_CA: ${OPENBALENA_ROOT_CA}
-      COOKIE_SESSION_SECRET: ${OPENBALENA_COOKIE_SESSION_SECRET}
-      DB_HOST: db
-      DB_PASSWORD: docker
-      DB_PORT: 5432
-      DB_USER: docker
-      DELTA_HOST: delta.${OPENBALENA_HOST_NAME}
-      DEVICE_CONFIG_OPENVPN_CA: ${OPENBALENA_VPN_CA_CHAIN}
-      DEVICE_CONFIG_SSH_AUTHORIZED_KEYS: ${OPENBALENA_SSH_AUTHORIZED_KEYS}
-      HOST: api.${OPENBALENA_HOST_NAME}
-      IMAGE_MAKER_URL: img.${OPENBALENA_HOST_NAME}
-      IMAGE_STORAGE_BUCKET: resin-production-img-cloudformation
-      IMAGE_STORAGE_PREFIX: images
-      IMAGE_STORAGE_ENDPOINT: s3.amazonaws.com
-      JSON_WEB_TOKEN_EXPIRY_MINUTES: 10080
-      JSON_WEB_TOKEN_SECRET: ${OPENBALENA_JWT_SECRET}
-      MIXPANEL_TOKEN: __unused__
-      PRODUCTION_MODE: "${OPENBALENA_PRODUCTION_MODE}"
-      PUBNUB_PUBLISH_KEY: __unused__
-      PUBNUB_SUBSCRIBE_KEY: __unused__
-      REDIS_HOST: redis
-      REDIS_PORT: 6379
-      REGISTRY2_HOST: registry.${OPENBALENA_HOST_NAME}
-      REGISTRY_HOST: registry.${OPENBALENA_HOST_NAME}
-      SENTRY_DSN: ""
-      TOKEN_AUTH_BUILDER_TOKEN: ${OPENBALENA_TOKEN_AUTH_BUILDER_TOKEN}
-      TOKEN_AUTH_CERT_ISSUER: api.${OPENBALENA_HOST_NAME}
-      TOKEN_AUTH_CERT_KEY: ${OPENBALENA_TOKEN_AUTH_KEY}
-      TOKEN_AUTH_CERT_KID: ${OPENBALENA_TOKEN_AUTH_KID}
-      TOKEN_AUTH_CERT_PUB: ${OPENBALENA_TOKEN_AUTH_PUB}
-      TOKEN_AUTH_JWT_ALGO: "ES256"
-      VPN_HOST: vpn.${OPENBALENA_HOST_NAME}
-      VPN_PORT: 443
-      VPN_SERVICE_API_KEY: ${OPENBALENA_VPN_SERVICE_API_KEY}
-      SUPERUSER_EMAIL: ${OPENBALENA_SUPERUSER_EMAIL}
-      SUPERUSER_PASSWORD: ${OPENBALENA_SUPERUSER_PASSWORD}
-
-  registry:
-    extends:
-      file: ./common.yml
-      service: component
-    image: balena/open-balena-registry:${OPENBALENA_REGISTRY_VERSION_TAG}
-    depends_on:
-      - s3
-      - redis
-    environment:
-      API_TOKENAUTH_CRT: ${OPENBALENA_TOKEN_AUTH_PUB}
-      BALENA_REGISTRY2_HOST: registry.${OPENBALENA_HOST_NAME}
-      BALENA_ROOT_CA: ${OPENBALENA_ROOT_CA}
-      BALENA_TOKEN_AUTH_ISSUER: api.${OPENBALENA_HOST_NAME}
-      BALENA_TOKEN_AUTH_REALM: https://api.${OPENBALENA_HOST_NAME}/auth/v1/token
-      COMMON_REGION: ${OPENBALENA_S3_REGION}
-      REGISTRY2_CACHE_ENABLED: "false"
-      REGISTRY2_CACHE_ADDR: 127.0.0.1:6379
-      REGISTRY2_CACHE_DB: 0
-      REGISTRY2_CACHE_MAXMEMORY_MB: 1024 # megabytes
-      REGISTRY2_CACHE_MAXMEMORY_POLICY: allkeys-lru
-      REGISTRY2_S3_REGION_ENDPOINT: ${OPENBALENA_S3_ENDPOINT}
-      REGISTRY2_S3_BUCKET: ${OPENBALENA_REGISTRY2_S3_BUCKET}
-      REGISTRY2_S3_KEY: ${OPENBALENA_S3_ACCESS_KEY}
-      REGISTRY2_S3_SECRET: ${OPENBALENA_S3_SECRET_KEY}
-      REGISTRY2_SECRETKEY: ${OPENBALENA_REGISTRY_SECRET_KEY}
-      REGISTRY2_STORAGEPATH: /data
-      REGISTRY2_DISABLE_REDIRECT: "false"
-
-  vpn:
-    extends:
-      file: ./common.yml
-      service: component
-    image: balena/open-balena-vpn:${OPENBALENA_VPN_VERSION_TAG}
-    depends_on:
-      - api
-    cap_add:
-      - NET_ADMIN
-    environment:
-      API_SERVICE_API_KEY: ${OPENBALENA_API_VPN_SERVICE_API_KEY}
-      BALENA_API_HOST: api.${OPENBALENA_HOST_NAME}
-      BALENA_ROOT_CA: ${OPENBALENA_ROOT_CA}
-      BALENA_VPN_PORT: 443
-      PRODUCTION_MODE: "${OPENBALENA_PRODUCTION_MODE}"
-      RESIN_VPN_GATEWAY: 10.2.0.1
-      SENTRY_DSN: ""
-      VPN_HAPROXY_USEPROXYPROTOCOL: "true"
-      VPN_OPENVPN_CA_CRT: ${OPENBALENA_VPN_CA}
-      VPN_OPENVPN_SERVER_CRT: ${OPENBALENA_VPN_SERVER_CRT}
-      VPN_OPENVPN_SERVER_DH: ${OPENBALENA_VPN_SERVER_DH}
-      VPN_OPENVPN_SERVER_KEY: ${OPENBALENA_VPN_SERVER_KEY}
-      VPN_SERVICE_API_KEY: ${OPENBALENA_VPN_SERVICE_API_KEY}
-
-  db:
-    extends:
-      file: ./common.yml
-      service: system
-    image: balena/open-balena-db:${OPENBALENA_DB_VERSION_TAG}
-    volumes:
-      - db:/var/lib/postgresql/data
-
-  s3:
-    extends:
-      file: ./common.yml
-      service: component
-    image: balena/open-balena-s3:${OPENBALENA_S3_VERSION_TAG}
-    volumes:
-      - s3:/export
-    environment:
-      S3_MINIO_ACCESS_KEY: ${OPENBALENA_S3_ACCESS_KEY}
-      S3_MINIO_SECRET_KEY: ${OPENBALENA_S3_SECRET_KEY}
-      BUCKETS: ${OPENBALENA_S3_BUCKETS}
-
-  redis:
-    extends:
-      file: ./common.yml
-      service: system
-    image: redis:alpine
-    volumes:
-      - redis:/data
-
-  haproxy:
-    extends:
-      file: ./common.yml
-      service: system
-    build: ../src/haproxy
-    depends_on:
-      - api
-      - cert-provider
-      - db
-      - s3
-      - redis
-      - registry
-      - vpn
-    ports:
-      - "80:80"
-      - "443:443"
-    expose:
-      - "222"
-      - "3128"
-      - "5432"
-      - "6379"
-    networks:
-      default:
-        aliases:
-          - api.${OPENBALENA_HOST_NAME}
-          - registry.${OPENBALENA_HOST_NAME}
-          - vpn.${OPENBALENA_HOST_NAME}
-          - db.${OPENBALENA_HOST_NAME}
-          - s3.${OPENBALENA_HOST_NAME}
-          - redis.${OPENBALENA_HOST_NAME}
-          - tunnel.${OPENBALENA_HOST_NAME}
-    environment:
-      BALENA_HAPROXY_CRT: ${OPENBALENA_ROOT_CRT}
-      BALENA_HAPROXY_KEY: ${OPENBALENA_ROOT_KEY}
-      BALENA_ROOT_CA: ${OPENBALENA_ROOT_CA}
-      HAPROXY_HOSTNAME: ${OPENBALENA_HOST_NAME}
-    volumes:
-      - certs:/certs:ro
-
-  cert-provider:
-    build: ../src/cert-provider
-    volumes:
-      - certs:/certs
-      - cert-provider:/usr/src/app/certs
-    environment:
-      ACTIVE: ${OPENBALENA_ACME_CERT_ENABLED}
-      DOMAINS: "api.${OPENBALENA_HOST_NAME},registry.${OPENBALENA_HOST_NAME},s3.${OPENBALENA_HOST_NAME},vpn.${OPENBALENA_HOST_NAME},tunnel.${OPENBALENA_HOST_NAME}"
-      OUTPUT_PEM: /certs/open-balena.pem

compose/template.yml

@@ -1,10 +0,0 @@
-# Project-specific config.
-#
-# All paths must be defined relative to `compose/services.yml` regardless of
-# the location of this file, i.e. refer to `my-open-balena-checkout/somedir`
-# as `../somedir`. This is because of the way docker-compose handles paths
-# when specifying multiple configs and open-balena always specifying
-# `compose/services.yml` as the "base" config.
-#
-# You may view the effective config with `scripts/compose config`.
-version: "2.0"

compose/versions

@@ -1,6 +0,0 @@
-export OPENBALENA_API_VERSION_TAG=v0.139.0
-export OPENBALENA_DB_VERSION_TAG=v4.1.0
-export OPENBALENA_MDNS_PUBLISHER_VERSION_TAG=v1.9.2
-export OPENBALENA_REGISTRY_VERSION_TAG=v2.16.1
-export OPENBALENA_S3_VERSION_TAG=v2.9.9
-export OPENBALENA_VPN_VERSION_TAG=v9.17.11

docker-compose.yml

@@ -0,0 +1,372 @@
+---
+version: '2.4'
+
+volumes:
+  builder-certs-ca: {}
+  builder-certs-client: {}
+  builder-data: {}
+  cert-manager-data: {}
+  certs-data: {}
+  db-data: {}
+  pki-data: {}
+  redis-data: {}
+  resin-data: {}
+  s3-data: {}
+
+x-default-healthcheck: &default-healthcheck
+  test: /usr/src/app/docker-hc
+  interval: 45s
+  timeout: 15s
+  retries: 3
+
+x-default-environment: &default-environment
+  # FIXME: hardcoded https://github.com/balena-io/open-balena-db/blob/master/create-resin-db.sh#L4
+  DB_NAME: resin
+  # FIXME: hardcoded https://github.com/balena-io/open-balena-db/blob/master/Dockerfile#L3-L4
+  DB_PASSWORD: docker
+  DB_USER: docker
+  LOG_LEVEL: DEBUG
+  PRODUCTION_MODE: 'false'
+
+x-default-healthcheck-trait: &with-default-healthcheck
+  healthcheck:
+    <<: *default-healthcheck
+
+x-default-volumes-trait: &with-default-volumes
+  volumes:
+    - certs-data:/certs
+    - resin-data:/balena
+
+x-default-privileges-trait: &with-default-privileges
+  cap_add:
+    - SYS_ADMIN
+    - SYS_RESOURCE
+  security_opt:
+    - apparmor=unconfined
+  tmpfs:
+    - /run
+    - /sys/fs/cgroup
+
+x-extended-privileges-trait: &with-extended-privileges
+  security_opt:
+    - apparmor=unconfined
+    - seccomp=unconfined
+
+x-all-privileges-trait: &with-all-privileges
+  privileged: true
+  cap_add:
+    - ALL
+
+x-network-privileges-trait: &with-network-privileges
+  cap_add:
+    - NET_ADMIN
+    - SYS_ADMIN
+    - SYS_RESOURCE
+
+x-base-service-definition: &base-service
+  restart: 'unless-stopped'
+  # for docker-compose only, no effect on balenaCloud
+  env_file:
+    - .env
+  tty: true  # send syastemd logs from containers to stdout
+
+services:
+  # https://github.com/balena-io/open-balena-api
+  api:
+    <<: [
+      *base-service,
+      *with-default-healthcheck,
+      *with-default-privileges,
+      *with-default-volumes,
+    ]
+    image: balena/open-balena-api:v25.1.24
+    depends_on:
+      - db
+      - redis
+      - s3
+    environment:
+      <<: *default-environment
+      CONTRACTS_PUBLIC_REPO_NAME: contracts
+      CONTRACTS_PUBLIC_REPO_OWNER: balena-io
+      DB_GENERAL_REPLICA_MAX_USES: 1000
+      DB_GENERAL_REPLICA_PORT: 5432
+      DB_HOST: db
+      DB_PORT: 5432
+      DB_STATE_REPLICA_MAX_USES: 1000
+      DB_STATE_REPLICA_PORT: 5432
+      DB_USER: docker
+      HOSTS_CONFIG: API_HOST:api,DB_HOST:db,DELTA_HOST:delta,HOST:api,REDIS_HOST:redis,TOKEN_AUTH_CERT_ISSUER:api,VPN_HOST:cloudlink,REGISTRY2_HOST:registry2
+      IMAGE_STORAGE_BUCKET: resin-production-img-cloudformation
+      IMAGE_STORAGE_ENDPOINT: s3.amazonaws.com
+      IMAGE_STORAGE_PREFIX: images
+      JSON_WEB_TOKEN_EXPIRY_MINUTES: 10080
+      NUM_WORKERS: 1
+      OAUTH_CALLBACK_PROTOCOL: https
+      PORT: 80
+      REDIS_HOST: redis:6379
+      REDIS_IS_CLUSTER: 'false'
+      TOKEN_AUTH_JWT_ALGO: ES256
+      TOKENS_CONFIG: API_SERVICE_API_KEY:hex,AUTH_RESINOS_REGISTRY_CODE:hex,COOKIE_SESSION_SECRET:hex,JSON_WEB_TOKEN_SECRET:hex,MIXPANEL_TOKEN:hex,SUPERUSER_PASSWORD:hex,TOKEN_AUTH_BUILDER_TOKEN:hex,VPN_GUEST_API_KEY:hex,VPN_SERVICE_API_KEY:hex,API_VPN_SERVICE_API_KEY:API_SERVICE_API_KEY,REGISTRY2_TOKEN:TOKEN_AUTH_BUILDER_TOKEN
+      TRUST_PROXY: 172.16.0.0/12
+      VPN_PORT: 443
+      WEBRESOURCES_S3_BUCKET: web-resources
+      WEBRESOURCES_S3_REGION: 'us-east-1'  # this is required for minio
+
+  # https://github.com/balena-io/open-balena-registry
+  registry:
+    <<: [
+      *base-service,
+      *with-default-healthcheck,
+      *with-default-privileges,
+    ]
+    image: balena/open-balena-registry:v2.39.60
+    volumes:
+      - certs-data:/certs
+      - resin-data:/balena
+    depends_on:
+      - redis
+      - s3
+    environment:
+      COMMON_REGION: open-balena
+      HOSTS_CONFIG: REGISTRY2_HOST:registry2,REGISTRY2_TOKEN_AUTH_ISSUER:api,REGISTRY2_TOKEN_AUTH_REALM:api
+      REGISTRY2_CACHE_ADDR: redis:6379
+      REGISTRY2_CACHE_DB: 1
+      REGISTRY2_CACHE_ENABLED: 'true'
+      REGISTRY2_S3_BUCKET: registry-data
+      REGISTRY2_STORAGEPATH: /data
+      TOKENS_CONFIG: REGISTRY2_SECRETKEY:hex
+
+  # https://github.com/balena-io/open-balena-vpn
+  vpn:
+    <<: [
+      *base-service,
+      *with-default-healthcheck,
+      *with-default-volumes,
+      # privileges in order from minimum to maximum
+      *with-network-privileges,
+      *with-default-privileges,
+    ]
+    image: balena/open-balena-vpn:v11.30.22
+    depends_on:
+      - api
+    environment:
+      HOSTS_CONFIG: VPN_HOST:cloudlink
+      TOKENS_CONFIG: ','
+      VPN_HAPROXY_USEPROXYPROTOCOL: 'true'
+      VPN_PORT: 443
+      # ensure correct service instance IP is registered with the API
+      VPN_SERVICE_REGISTER_INTERFACE: eth0
+
+  # https://github.com/balena-io/open-balena-db
+  db:
+    <<: *base-service
+    image: balena/open-balena-db:v5.2.2
+    volumes:
+      - db-data:/var/lib/postgresql/data
+    environment:
+      <<: *default-environment
+    healthcheck:
+      test: pg_isready -U "$${DB_USER}" -d "$${DB_NAME}"
+
+  # https://github.com/balena-io/open-balena-s3
+  s3:
+    <<: [
+      *base-service,
+      *with-default-healthcheck,
+      *with-default-privileges,
+    ]
+    image: balena/open-balena-s3:v2.28.47
+    volumes:
+      - s3-data:/export
+      - certs-data:/certs
+      - resin-data:/balena
+    environment:
+      BUCKETS: registry-data;web-resources
+      HOSTS_CONFIG: REGISTRY2_S3_REGION_ENDPOINT:s3,WEBRESOURCES_S3_HOST:s3
+      TOKENS_CONFIG: REGISTRY2_S3_KEY:hex,REGISTRY2_S3_SECRET:hex,S3_MINIO_ACCESS_KEY:REGISTRY2_S3_KEY,S3_MINIO_SECRET_KEY:REGISTRY2_S3_SECRET,WEBRESOURCES_S3_ACCESS_KEY:REGISTRY2_S3_KEY,WEBRESOURCES_S3_SECRET_KEY:REGISTRY2_S3_SECRET
+
+  # https://hub.docker.com/_/redis
+  redis:
+    <<: *base-service
+    # https://redis.io/blog/what-redis-license-change-means-for-our-managed-service-providers/
+    image: redis:7.2-alpine
+    volumes:
+      - redis-data:/data
+    healthcheck:
+      <<: *default-healthcheck
+      test: echo INFO | redis-cli | grep redis_version
+
+  # https://github.com/balena-io/open-balena-haproxy
+  haproxy:
+    <<: [
+      *base-service,
+      *with-default-privileges,
+      *with-default-volumes,
+    ]
+    build: src/haproxy
+    sysctls:
+      # https://github.com/docker-library/haproxy/issues/160
+      net.ipv4.ip_unprivileged_port_start: 0
+    healthcheck:
+      <<: *default-healthcheck
+      test: true | openssl s_client -connect localhost:443
+    ports:
+      # haproxy/http
+      - '80:80/tcp'
+      # haproxy/tcp-router
+      - '443:443/tcp'
+      # haproxy/stats
+      - '1936:1936/tcp'
+    environment:
+      LOGLEVEL: info
+
+  # dynamically configure Docker network aliases based on DNS_TLD and ALIAS list
+  # allows DNS resolution from systemd-less images on the Docker network
+  haproxy-sidecar:
+    <<: *base-service
+    build: src/haproxy-sidecar
+    volumes:
+      - /var/run/docker.sock:/host/run/docker.sock
+    environment:
+      DOCKER_HOST: unix:///host/run/docker.sock
+      # resolved internally as {{service}}.{{dns-tld-without-balena-device-uuid}} to haproxy service
+      ALIASES: api,ca,cloudlink,db,delta,logs,redis,registry2,s3,stats,tunnel
+    labels:
+      io.balena.features.balena-socket: 1
+      io.balena.features.supervisor-api	: 1
+
+  # https://github.com/balena-io/cert-manager
+  # https://certbot.eff.org/docs/using.html
+  # https://certbot-dns-cloudflare.readthedocs.io/
+  cert-manager:
+    <<: *base-service
+    build: src/cert-manager
+    volumes:
+      - cert-manager-data:/etc/letsencrypt
+      - certs-data:/certs
+      - resin-data:/balena
+    depends_on:
+      - balena-ca
+    environment:
+      # wildcard certificate for reverse proxy
+      SSH_KEY_NAMES: ','
+      SUBJECT_ALTERNATE_NAMES: '*'
+    labels:
+      io.balena.features.balena-api: 1
+      io.balena.features.supervisor-api: 1
+
+  # https://github.com/balena-io/ca-private
+  # https://github.com/cloudflare/cfssl/blob/master/doc/api/intro.txt
+  balena-ca:
+    <<: *base-service
+    image: balena/ca-private:v0.0.14
+    volumes:
+      - pki-data:/pki
+      - certs-data:/certs
+      - resin-data:/balena
+    healthcheck:
+      test: curl --silent -I --fail localhost:8888
+      interval: 60s
+      timeout: 60s
+      retries: 10
+    labels:
+      # future expansion
+      io.balena.features.balena-api: 1
+      io.balena.features.supervisor-api: 1
+
+
+
+  # --- the following are not required for runtime operation of openBalena
+
+  # only relevant when running in AWS/EC2
+  tag-sidecar:
+    build: src/tag-sidecar
+    restart: 'no'
+    environment:
+      ENABLED: 'true'
+    labels:
+      io.balena.features.balena-api: 1
+
+  # Software Under Test (SUT) tests orchestrator
+  sut:
+    <<: [
+      *base-service,
+      *with-extended-privileges,
+      *with-network-privileges,
+    ]
+    build: src/balena-tests
+    command: /usr/sbin/balena.sh
+    environment:
+      DOCKER_CERT_PATH: /docker-pki/client
+      DOCKER_HOST: docker:2376
+      DOCKER_TLS_VERIFY: 'true'
+      GUEST_IMAGE: /balena/balena.img
+    volumes:
+      - builder-certs-client:/docker-pki/client
+      - certs-data:/certs
+      - resin-data:/balena
+    labels:
+      io.balena.features.balena-api: 1
+      io.balena.features.supervisor-api: 1
+    restart: 'no'
+
+  # virtual Device Under Test (DUT)
+  dut:
+    <<: [
+      *base-service,
+      *with-extended-privileges,
+      *with-network-privileges,
+    ]
+    # https://hub.docker.com/r/qemux/qemu-docker
+    # https://github.com/qemus/qemu-docker
+    build: src/test-device
+    entrypoint:
+      - /bin/sh
+      - -c
+    command:
+      - /usr/sbin/balena.sh
+    environment:
+      GUEST_IMAGE: /balena/balena.img
+      MEMORY: 3072M
+      CPU: 4
+    volumes:
+      - resin-data:/balena
+    devices:
+      - /dev/net/tun
+    restart: 'no'
+
+  # https://hub.docker.com/_/docker
+  # pseudo(builder) service for balena-tests
+  docker:
+    <<: [
+      *base-service,
+      *with-extended-privileges,
+      *with-network-privileges,
+    ]
+    image: docker:dind
+    entrypoint:
+      - /bin/sh
+      - -c
+    command:
+      - |
+        set -x
+
+        cp /certs/root-ca.pem /certs/server-ca.pem /usr/local/share/ca-certificates/ \
+          && update-ca-certificates
+
+        exec /usr/local/bin/dockerd-entrypoint.sh
+    volumes:
+      - /sys:/sys
+      - builder-certs-ca:/docker-pki/ca
+      - builder-certs-client:/docker-pki/client
+      - builder-data:/var/lib/docker
+      - certs-data:/certs
+    environment:
+      DOCKER_TLS_CERTDIR: /docker-pki
+    healthcheck:
+      test: docker system info
+      interval: 60s
+      timeout: 60s
+      retries: 5
+    labels:
+      io.balena.features.sysfs: 1

docs/getting-started.md

@@ -0,0 +1,485 @@
+# openBalena Getting Started Guide
+
+This guide will walk you through the steps of deploying an openBalena server, that
+together with the balena CLI, will enable you to create and manage a fleet of devices
+running on your own infrastructure, on premises or in the cloud. The openBalena servers
+must be reachable by the devices, which is easiest to achieve with cloud providers like
+AWS, Google Cloud, Digital Ocean and others.
+
+This guide assumes a setup with two separate machines:
+
+- A _server_, running Linux with at least 2GB of memory. These instructions were tested
+  with Ubuntu 20.04, 22.04 and 24.04 x64 servers. The server must have a working
+  installation of [Docker Engine] and you must have root permissions.
+- A _local machine_, running Linux, Windows or macOS where the balena CLI runs (as a
+  client to the openBalena server). The local machine must also have a working
+  installation of [Docker] so that application images can be built and deployed to your
+  device. It is also possible to use [balenaEngine] on a [balenaOS] device instead of
+  Docker.
+
+Additionally, a _device type_ and compatible flash media supported by [balenaOS]
+(e.g. Raspberry Pi) are required to complete the provisioning demo. Ensure the correct
+power supply is available to power this device.
+
+## Domain Configuration
+
+The following DNS records must be configured to point to the openBalena server prior to
+configuration:
+
+```text
+api.mydomain.com
+ca.mydomain.com
+cloudlink.mydomain.com
+logs.mydomain.com
+ocsp.mydomain.com
+registry2.mydomain.com
+s3.mydomain.com
+tunnel.mydomain.com
+```
+
+Alternatively you may consider adding a single wildcard DNS record `*.mydomain.com`.
+
+Check with your Internet domain name registrar for instructions on how to obtain a domain
+name and configure records.
+
+## Install openBalena on the server
+
+1. First [Change cgroup version] to v1 for compatibility with systemd in containers on
+   modern Linux distributions, where cgroups v2 are enabled by default:
+
+    ```bash
+    source /etc/default/grub
+    sudo sed -i '/GRUB_CMDLINE_LINUX/d' /etc/default/grub
+    echo GRUB_CMDLINE_LINUX=$(printf '\"%s systemd.unified_cgroup_hierarchy=0\"\n' "${GRUB_CMDLINE_LINUX}") \
+      | sudo tee -a /etc/default/grub
+    sudo update-grub
+    sudo reboot
+    ```
+
+2. Ensure cgroups v2 is disabled
+
+    ```bash
+    if [ ! -f /sys/fs/cgroup/cgroup.controllers ]; then
+        echo "cgroups v2 is disabled"
+    else
+        echo "cgroups v2 is enabled"
+    fi
+    ```
+
+3. Now, install or update essential software:
+
+    ```bash
+    sudo apt-get update && sudo apt-get install -y make openssl git jq
+    ```
+
+4. Install Docker Engine
+
+    ```bash
+    which docker || curl -fsSL https://get.docker.com | sh -
+    ```
+
+5. Create a new user with appropriate permissions:
+
+    ```bash
+    sudo useradd -s /bin/bash -m -G docker,sudo balena
+    echo 'balena ALL=(ALL) NOPASSWD: ALL' | sudo tee /etc/sudoers.d/balena
+    ```
+
+6. Switch user:
+
+   ```bash
+   sudo su balena
+   ```
+
+7. Clone the openBalena repository and change directory:
+
+   ```bash
+   git clone https://github.com/balena-io/open-balena.git ~/open-balena
+   cd ~/open-balena
+   ```
+
+8. Start the server on your domain name:
+
+   ```bash
+   export DNS_TLD=mydomain.com
+   make up
+   ```
+
+   Note down `SUPERUSER_EMAIL` and `SUPERUSER_PASSWORD` values to be used later.
+
+9. Tail the logs of the containers with:
+
+   ```bash
+   docker compose logs -f api
+   ```
+
+   Replace `api` with the name of any one of the services from the [composition].
+
+10. The server can be stopped with:
+
+   ```bash
+   make down
+   ```
+
+   The server can also be restarted using `make restart`.
+
+To update openBalena, run:
+
+```bash
+make update
+```
+
+### Test the openBalena server
+
+To confirm that everything is running correctly, try a simple request from the local
+machine to the server after registering its CA certificate(s) with the host:
+
+```bash
+make self-signed
+make verify
+```
+
+Note, if you've previously stopped the server with `make down`, run `make up` again first.
+
+Congratulations! The openBalena server is up and running. The next step is to setup your
+local machine to use this server, provision a device and deploy a small project.
+
+### Install self-signed certificates on the local machine.
+
+The installation of the openBalena server produces a self-signed certificate by default,
+which must be trusted by all devices communicating with it. This type of configuration is
+not recommended for production deployments, skip to [SSL Configuration](#ssl-configuration)
+instead.
+
+The root CA bundle can be found at `.balena/ca-${DNS_TLD}.pem` on the server. Follow the
+steps below for your specific local machine platform after manually copying it across.
+
+#### Linux:
+
+```bash
+sudo cp ca.pem /usr/local/share/ca-certificates/
+sudo update-ca-certificates
+sudo systemctl restart docker
+```
+
+#### macOS:
+
+```bash
+sudo security add-trusted-cert -d \
+  -r trustRoot \
+  -k /Library/Keychains/System.keychain \
+  ca.pem
+
+curl http://localhost/engine/restart \
+  -H 'Content-Type: application/json' \
+  -d '{"openContainerView": true}' \
+  --unix-socket ~/Library/Containers/com.docker.docker/Data/backend.sock
+```
+
+#### Windows:
+
+```PowerShell
+certutil -addstore -f "ROOT" ca.pem
+Stop-Service -Name Docker
+Start-Service -Name Docker
+```
+
+### SSL Configuration
+
+opeBalena server now uses automatic SSL configuration via ACME [DNS-01] challenge[^1]. Support
+for the following DNS providers is currently implemented:
+
+* Cloudflare
+* Gandi
+
+#### Cloudflare
+
+Obtain a Cloudflare API token with write access to your openBalena domain name records:
+
+```bash
+export ACME_EMAIL=acme@mydomain.com
+export CLOUDFLARE_API_TOKEN={{token}}
+```
+
+#### Gandi
+
+Obtain a Gandi API token with write access to your openBalena domain name records:
+
+```bash
+export ACME_EMAIL=acme@mydomain.com
+export GANDI_API_TOKEN={{token}}
+```
+
+#### Re-configure and test the server
+
+```bash
+make auto-pki
+make verify
+```
+
+#### Custom SSL
+
+openBalena server also supports custom/manual TLS configuration. You must supply your own
+SSL certificate, private key and a full certificate signing chain. A wildcard SSL
+certificate covering the whole domain is recommended.
+
+1. After obtaining your certificate, run the following commands on openBalena server:
+
+```bash
+export HAPROXY_CRT="{{ base64 encoded server certificate }}"
+export ROOT_CA="{{ .. intermediate certificates }}"
+export HAPROXY_KEY="{{ .. private key }}"
+```
+
+Pipe the plaintext via `.. | openssl base64 -A` to encode.
+
+2. Re-configure and test the server:
+
+```bash
+make pki-custom
+make verify
+```
+
+### Install the balena CLI on the local machine
+
+Follow the [balena CLI installation instructions] to install the balena CLI on the local
+machine.
+
+By default, the CLI targets the balenaCloud servers at `balena-cloud.com`, and
+needs to be configured to target the openBalena server instead. Add the following
+line to the CLI's configuration file, replacing `"mydomain.com"` with the domain
+name of the openBalena server:
+
+```yaml
+balenaUrl: 'mydomain.com'
+```
+
+The CLI configuration file can be found at:
+
+- On Linux or macOS: `~/.balenarc.yml`
+- On Windows: `%UserProfile%\_balenarc.yml`
+
+If the file does not already exist, just create it. Alternatively, `BALENARC_BALENA_URL`
+environment variable can be set to point to `"mydomain.com"`.
+
+Wrapping up the CLI installation, set an environment variable that points to the
+root certificate copied previously on the local machine. This step is to ensure
+the CLI can securely interact with the openBalena server when running self-signed PKI.
+This step can be skipped if the server is operating with publicly trusted PKI.
+
+| Shell              | Command                                        |
+| ------------------ | ---------------------------------------------- |
+| bash               | `export NODE_EXTRA_CA_CERTS='/path/to/ca.pem'` |
+| Windows cmd.exe    | `set NODE_EXTRA_CA_CERTS=C:\path\to\ca.pem`    |
+| Windows PowerShell | `$Env:NODE_EXTRA_CA_CERTS="C:\path\to\ca.pem"` |
+
+### Deploy an application
+
+The commands below should be run on a terminal on the local machine (where the
+balena CLI is installed). Ensure that the `NODE_EXTRA_CA_CERTS` environment
+variable is set, as discussed above.
+
+#### Login to openBalena
+
+Run `balena login`, select `Credentials` and use `SUPERUSER_EMAIL` and
+`SUPERUSER_PASSWORD` generated during `make up` step to login to the openBalena server.
+At any time, `balena whoami` command may be used to check which server the CLI is
+authenticated with.
+
+#### Create an application
+
+Create a new application with `balena fleet create myApp`. Select the application's
+default device type with the interactive prompt. The examples in this guide assume
+a Raspberry Pi 3.
+
+An application contains devices that share the same architecture (such as ARM or Intel),
+and also contains code releases that are deployed to the devices. When a device is
+provisioned, it is added to an application, but can be migrated to another application at
+any time. There is no limit to the number of applications that can be created or to the
+number of devices that can be provisioned.
+
+At any time, the server can be queried for all the applications it knows about
+with the following command:
+
+```bash
+balena fleets
+ Id App name Slug        Device type  Device count Online devices
+ ── ──────── ─────────── ──────────── ──────────── ──────────────
+ 1  myApp    admin/myapp raspberrypi3 0            0
+```
+
+#### Provision a new device
+
+Once we have an application, it’s time to start provisioning devices. To do this,
+first download a [balenaOS] image for your device. For this example we are using a
+Raspberry Pi 3.
+
+Unzip the downloaded image and use the balena CLI to configure it:
+
+```bash
+balena os configure --dev --fleet myApp ~/Downloads/raspberrypi3-5.2.8-v16.1.10.img
+```
+
+Flash the configured image to an SD card using [Etcher] or balena CLI:
+
+```bash
+sudo balena local flash ~/Downloads/raspberrypi3-5.2.8-v16.1.10.img
+```
+
+Insert the SD card into the device and power it on. The device will register with the
+openBalena server and after about two minutes will be inspectable:
+
+```bash
+balena devices
+ID UUID    DEVICE NAME DEVICE TYPE  FLEET       STATUS IS ONLINE SUPERVISOR VERSION OS VERSION
+1  560dcc2 quiet-rock  raspberrypi3 admin/myapp Idle   true      16.1.10            balenaOS 5.2.8
+
+balena device 560dcc2
+== WANDERING RAIN
+ID:                    1
+DEVICE TYPE:           raspberrypi3
+STATUS:                idle
+IS ONLINE:             true
+IP ADDRESS:            192.168.1.42
+MAC ADDRESS:           B8:27:DE:AD:BE:EF
+FLEET:                 admin/myapp
+LAST SEEN:             1977-08-20T14:29:00.042Z
+UUID:                  560dcc24b221c8a264d5bd981284801f
+COMMIT:                N/a
+SUPERVISOR VERSION:    16.1.10
+IS WEB ACCESSIBLE:     false
+OS VERSION:            balenaOS 5.2.8
+DASHBOARD URL:         https://dashboard.mydomain.com/devices/560dcc24b221c8a264d5bd981284801f/summary
+CPU USAGE PERCENT:     2
+CPU TEMP C:            39
+CPU ID:                00000000335956af
+MEMORY USAGE MB:       140
+MEMORY TOTAL MB:       971
+MEMORY USAGE PERCENT:  14
+STORAGE BLOCK DEVICE:  /dev/mmcblk0p6
+STORAGE USAGE MB:      76
+STORAGE TOTAL MB:      14121
+STORAGE USAGE PERCENT: 1
+```
+
+Note, even though the dashboard URL is populated, there is no dashboard service in
+openBalena.
+
+It's time to deploy code to the device.
+
+#### Deploy a project
+
+Application release images are built on the local machine using the balena CLI. Ensure the
+root certificate has been correctly installed on the local machine, as discussed above.
+
+Let's create a trivial project that logs "Idling...". On an empty directory, create a new
+file named `Dockerfile.template` with the following contents:
+
+```dockerfile
+FROM balenalib/%%BALENA_MACHINE_NAME%%-alpine
+
+CMD [ "balena-idle" ]
+```
+
+Then build and deploy the project with:
+
+```bash
+balena deploy --noparent-check myApp
+```
+
+The project will have been successfully built when a friendly unicorn appears in the
+terminal:
+
+```bash
+[Info]    No "docker-compose.yml" file found at "~/open-balena/balena-idle"
+[Info]    Creating default composition with source: "~/open-balena/balena-idle"
+[Info]    Everything is up to date (use --build to force a rebuild)
+[Info]    Creating release...
+[Info]    Pushing images to registry...
+[Info]    Saving release...
+[Success] Deploy succeeded!
+[Success] Release: 50be7bdb0ea6819c91a5dd7bcd7635ad
+
+                            \
+                             \
+                              \\
+                               \\
+                                >\/7
+                            _.-(6'  \
+                           (=___._/` \
+                                )  \ |
+                               /   / |
+                              /    > /
+                             j    < _\
+                         _.-' :      ``.
+                         \ r=._\        `.
+                        <`\\_  \         .`-.
+                         \ r-7  `-. ._  ' .  `\
+                          \`,      `-.`7  7)   )
+                           \/         \|  \'  / `-._
+                                      ||    .'
+                                       \\  (
+                                        >\  >
+                                    ,.-' >.'
+                                   <.'_.''
+                                     <'
+```
+
+This command packages up the local directory, creates a new Docker image from it and
+pushes it to the openBalena server. In turn, the server will deploy it to all provisioned
+devices and within a couple of minutes, they will all run the new release. Logs can be
+viewed with:
+
+```bash
+balena logs --tail 560dcc2
+[Logs]    [2024-05-02T15:59:31.383Z] Supervisor starting
+[Logs]    [2024-05-02T15:59:37.552Z] Applying configuration change {"SUPERVISOR_VPN_CONTROL":"true"}
+[Logs]    [2024-05-02T15:59:37.599Z] Applied configuration change {"SUPERVISOR_VPN_CONTROL":"true"}
+[Logs]    [2024-05-02T15:59:40.331Z] Creating network 'default'
+[Logs]    [2024-05-02T16:11:15.331Z] Supervisor starting
+[Logs]    [2024-05-02T16:44:08.199Z] Creating volume 'resin-data'
+[Logs]    [2024-05-02T16:44:08.572Z] Downloading image 'registry2.mydomain.com/v2/…
+…
+[Logs]    [2024-05-02T16:44:37.200Z] [main] Idling...
+[Logs]    [2024-05-02T16:44:37.200Z] [main] Idling...
+```
+
+Enjoy Balenafying All the Things!
+
+## Next steps
+
+- Try out [local mode], which allows you to build and sync code to your device locally for
+  rapid development.
+- Develop an application with [multiple containers] to provide a more modular approach to
+  application management.
+- Manage your device fleet with the use of [configuration] and [environment] variables.
+- Explore our [example projects] to give you an idea of more things you can do with
+  balena.
+- If you find yourself stuck or confused, help is just [a click away].
+- Pin selected devices to selected code releases using [sample scripts].
+- To change the superuser password after setting the credentials, follow this [forum post]
+
+
+[^1]: If DNS validation is not an option, [acme.sh] or [certbot] can be used to manually
+issue a certificate, which can then be set using the [custom SSL](#custom-ssl) workflow.
+
+
+[local mode]: https://www.balena.io/docs/learn/develop/local-mode
+[multiple containers]: https://www.balena.io/docs/learn/develop/multicontainer
+[configuration]: https://www.balena.io/docs/learn/manage/configuration
+[environment]: https://www.balena.io/docs/learn/manage/serv-vars
+[example projects]: https://balena.io/blog/tags/etcher-featured
+[a click away]: https://www.balena.io/support
+[sample scripts]: https://github.com/balena-io-examples/staged-releases
+[forum post]: https://forums.balena.io/t/upate-superuser-password/4738/6
+[balena CLI installation instructions]: https://github.com/balena-io/balena-cli/blob/master/INSTALL.md
+[Etcher]: https://balena.io/etcher
+[balenaOS]: https://balena.io/os/#download
+[balenaEngine]: https://www.balena.io/engine
+[Docker]: https://docs.docker.com/get-docker
+[Docker Engine]: https://docs.docker.com/engine/install
+[Change cgroup version]: https://docs.docker.com/config/containers/runmetrics/#changing-cgroup-version
+[composition]: https://github.com/balena-io/open-balena/blob/master/docker-compose.yml
+[DNS-01]: https://letsencrypt.org/docs/challenge-types/#dns-01-challenge
+[acme.sh]: https://github.com/acmesh-official/acme.sh
+[certbot]: https://certbot.eff.org/

repo.yml

@@ -1,5 +1,5 @@
-type: "generic"
+---
-reviewers: 1
+type: generic
 upstream:
   - repo: open-balena-api
     url: https://github.com/balena-io/open-balena-api
@@ -11,5 +11,3 @@ upstream:
     url: https://github.com/balena-io/open-balena-db
   - repo: open-balena-s3
     url: https://github.com/balena-io/open-balena-s3
-  - repo: balena-mdns-publisher
-    url: https://github.com/balena-io/balena-mdns-publisher

scripts/_keyid.js

@@ -1,79 +0,0 @@
-'use strict';
-
-var crypto = require('crypto');
-var fs = require('fs');
-
-var base32 = (function() {
-  // Extracted from https://github.com/chrisumbel/thirty-two
-  // to avoid having to install packages for this script.
-  var charTable = "ABCDEFGHIJKLMNOPQRSTUVWXYZ234567";
-  var byteTable = [
-    0xff, 0xff, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f,
-    0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-    0xff, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06,
-    0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e,
-    0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16,
-    0x17, 0x18, 0x19, 0xff, 0xff, 0xff, 0xff, 0xff,
-    0xff, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06,
-    0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e,
-    0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16,
-    0x17, 0x18, 0x19, 0xff, 0xff, 0xff, 0xff, 0xff
-  ];
-
-  function quintetCount(buff) {
-    var quintets = Math.floor(buff.length / 5);
-    return buff.length % 5 == 0 ? quintets: quintets + 1;
-  }
-
-  return function(plain) {
-    if (!Buffer.isBuffer(plain)) {
-      plain = new Buffer(plain);
-    }
-    var i = 0;
-    var j = 0;
-    var shiftIndex = 0;
-    var digit = 0;
-    var encoded = new Buffer(quintetCount(plain) * 8);
-
-    /* byte by byte isn't as pretty as quintet by quintet but tests a bit
-      faster. will have to revisit. */
-    while(i < plain.length) {
-      var current = plain[i];
-
-      if(shiftIndex > 3) {
-        digit = current & (0xff >> shiftIndex);
-        shiftIndex = (shiftIndex + 5) % 8;
-        digit = (digit << shiftIndex) | ((i + 1 < plain.length) ?
-          plain[i + 1] : 0) >> (8 - shiftIndex);
-        i++;
-      } else {
-        digit = (current >> (8 - (shiftIndex + 5))) & 0x1f;
-        shiftIndex = (shiftIndex + 5) % 8;
-        if(shiftIndex == 0) i++;
-      }
-
-      encoded[j] = charTable.charCodeAt(digit);
-      j++;
-    }
-
-    for (i = j; i < encoded.length; i++) {
-      encoded[i] = 0x3d; //'='.charCodeAt(0)
-    }
-    return encoded;
-  }
-})();
-
-function joseKeyId(der) {
-  var hasher = crypto.createHash('sha256');
-  hasher.update(der);
-  var b32 = base32(hasher.digest().slice(0, 30)).toString('ascii');
-  var chunks = [];
-  for (var i = 0; i < b32.length; i += 4) {
-    chunks.push(b32.substr(i, 4));
-  }
-  return chunks.join(':');
-}
-
-var derFilePath = process.argv[2];
-var der = fs.readFileSync(derFilePath);
-process.stdout.write(joseKeyId(der));

scripts/_realpath

@@ -1,35 +0,0 @@
-#!/bin/bash -e
-
-echo_error() {
-    local RED=`tput setaf 1`
-    local RESET=`tput sgr0`
-    echo "${RED}ERROR: ${1}${RESET}"
-}
-
-REALPATH=
-REALPATHS=(
-    'realpath'
-    'grealpath'
-    'greadlink -f'
-)
-for cmd in "${REALPATHS[@]}"; do
-if command -v "${cmd%% *}" &>/dev/null; then
-    REALPATH="${cmd}"
-    break
-fi
-done
-
-if [ -z "${REALPATH}" ]; then
-    echo_error 'Unable to find suitable command for realpath.'
-    if [ $(uname) == 'Darwin' ]; then
-        echo 'GNU coreutils are required to build openBalena on macOS. To install with brew, run'
-        echo ''
-        echo '    brew install coreutils'
-        echo ''
-    fi
-    exit 1
-fi
-
-realpath() {
-  echo $(command ${REALPATH} "$@")
-}

scripts/compose

@@ -1,40 +0,0 @@
-#!/bin/bash -e
-
-source "${BASH_SOURCE%/*}/_realpath"
-
-CMD="$(realpath "$0")"
-DIR="$(dirname "${CMD}")"
-BASE_DIR="$(dirname "${DIR}")"
-CONFIG_DIR="${BASE_DIR}/config"
-
-echo_bold() {
-  printf "\\033[1m%s\\033[0m\\n" "$@"
-}
-
-VERSIONS_FILE="${BASE_DIR}/compose/versions"
-if [ ! -f "$VERSIONS_FILE" ]; then
-  echo_bold "No service versions defined in ${VERSIONS_FILE}"
-  exit 1
-fi
-
-ENV_FILE="${CONFIG_DIR}/activate"
-if [ ! -f "$ENV_FILE" ]; then
-  echo_bold 'No configuration found; please create one first with: ./scripts/quickstart'
-  echo_bold 'See README.md for help.'
-  exit 1
-fi
-
-source "${ENV_FILE}"
-
-# only include the MDNS publisher IF the domain is valid...
-if [ ${OPENBALENA_HOST_NAME: -6} == ".local" ]; then
-  INCLUDE_MDNS="-f ${BASE_DIR}/compose/mdns.yml"
-fi
-
-# shellcheck source=/dev/null
-source "${VERSIONS_FILE}"; docker-compose \
-  --project-name 'openbalena' \
-  -f "${BASE_DIR}/compose/services.yml" \
-  ${INCLUDE_MDNS} \
-  -f "${CONFIG_DIR}/docker-compose.yml" \
-  "$@"

scripts/create-superuser

@@ -1,36 +0,0 @@
-#!/bin/bash -e
-
-usage() {
-  echo "usage: $0 EMAIL PASSWORD"
-  echo
-  echo 'Create the superuser account with the given email and password.'
-  echo
-  echo 'The instance must already be running in the background. You can '
-  echo 'start it with: ./scripts/compose up -d'
-}
-
-if [[ -z "$1" || -z "$2" ]]; then
-  usage
-  exit 1
-fi
-
-echo_bold() {
-  printf "\\033[1m%s\\033[0m\\n" "${@}"
-}
-
-source "${BASH_SOURCE%/*}/_realpath"
-
-CMD="$(realpath "$0")"
-DIR="$(dirname "${CMD}")"
-FIG="${DIR}/compose"
-
-EMAIL="$1"
-PASSWORD="$2"
-
-"${FIG}" exec api /bin/bash -c \
-  'export $(grep -v "^#" config/env | xargs -d "\n"); node index.js create-superuser root '${EMAIL}' '${PASSWORD}'' \
-  >/dev/null \
-  || (echo 'Failed to create superuser; please ensure the instance is running and that no superuser has been created before.' && exit 1)
-
-echo_bold "==> Success! Superuser created with email: ${EMAIL}"
-echo "  - You may now login with: balena login --credentials --email ${EMAIL}"

scripts/gen-root-ca

@@ -1,35 +0,0 @@
-#!/bin/bash -e
-
-usage() {
-  echo "usage: $0 COMMON_NAME [OUT]"
-  echo
-  echo "  COMMON_NAME   the domain name the certificate is valid for, eg. example.com"
-  echo "  OUT           path to output directory generated files will be placed in"
-  echo
-}
-
-if [ -z "$1" ]; then
-  usage
-  exit 1
-fi
-
-CMD="$(realpath "$0")"
-DIR="$(dirname "${CMD}")"
-
-CN="$1"
-OUT="$(realpath "${2:-.}")"
-
-# shellcheck source=scripts/ssl-common.sh
-source "${DIR}/ssl-common.sh"
-
-ROOT_CA="${ROOT_PKI}/ca.crt"
-
-if [ ! -f $ROOT_CA ]; then
-  # Create a secret key and CA file for the self-signed CA
-  "$easyrsa_bin" --pki-dir="${ROOT_PKI}" init-pki 2>/dev/null
-  "$easyrsa_bin" --pki-dir="${ROOT_PKI}" --days="${CA_EXPIRY_DAYS}" --req-cn="ca.${CN}" build-ca nopass 2>/dev/null
-
-  # update indexes and generate CRLs
-  "$easyrsa_bin" --pki-dir="${ROOT_PKI}" update-db 2>/dev/null
-  "$easyrsa_bin" --pki-dir="${ROOT_PKI}" gen-crl 2>/dev/null
-fi

scripts/gen-root-cert

@@ -1,36 +0,0 @@
-#!/bin/bash -e
-
-usage() {
-  echo "usage: $0 COMMON_NAME [OUT]"
-  echo
-  echo "  COMMON_NAME   the domain name the certificate is valid for, eg. example.com"
-  echo "  OUT           path to output directory generated files will be placed in"
-  echo
-}
-
-if [ -z "$1" ]; then
-  usage
-  exit 1
-fi
-
-CMD="$(realpath "$0")"
-DIR="$(dirname "${CMD}")"
-
-CN="$1"
-OUT="$(realpath "${2:-.}")"
-
-# shellcheck source=scripts/ssl-common.sh
-source "${DIR}/ssl-common.sh"
-
-ROOT_CRT="${ROOT_PKI}"'/issued/*.'"${CN}"'.crt'
-ROOT_KEY="${ROOT_PKI}"'/private/*.'"${CN}"'.key'
-
-if [ ! -f $ROOT_CRT ] || [ ! -f $ROOT_KEY ]; then
-  rm -f $ROOT_CRT $ROOT_KEY
-  # generate default CSR and sign (root + wildcard)
-  "$easyrsa_bin" --pki-dir="${ROOT_PKI}" --days="${CRT_EXPIRY_DAYS}" --subject-alt-name="DNS:*.${CN}" build-server-full "*.${CN}" nopass 2>/dev/null
-
-  # update indexes and generate CRLs
-  "$easyrsa_bin" --pki-dir="${ROOT_PKI}" update-db 2>/dev/null
-  "$easyrsa_bin" --pki-dir="${ROOT_PKI}" gen-crl 2>/dev/null
-fi;

scripts/gen-token-auth-cert

@@ -1,53 +0,0 @@
-#!/bin/bash -e
-
-usage() {
-  echo "usage: $0 COMMON_NAME [OUT]"
-  echo
-  echo "  COMMON_NAME   the domain name the certificate is valid for, eg. example.com"
-  echo "  OUT           path to output directory generated files will be placed in"
-  echo
-}
-
-if [ -z "$1" ]; then
-  usage
-  exit 1
-fi
-
-CMD="$(realpath "$0")"
-DIR="$(dirname "${CMD}")"
-
-CN="$1"
-OUT="$(realpath "${2:-.}")"
-
-# shellcheck source=scripts/ssl-common.sh
-source "${DIR}/ssl-common.sh"
-
-CERT_DIR="${OUT}/api"
-CERT_FILE="${CERT_DIR}/api.${CN}"
-
-keyid() {
-  # NodeJS is installed as `nodejs` in some distros, `node` in others.
-  node_bin="$(command -v nodejs 2>/dev/null || command -v node 2>/dev/null || true)"
-  if [ -z "$node_bin" ]; then
-    echo >&2 'NodeJS is required but not installed. Aborting.'
-    exit 1
-  fi
-  # Recent Node versions complain about `new Buffer()` being deprecated
-  # but the alternative is not available to older versions. Silence the
-  # warning but use the deprecated form to allow greater compatibility.
-  "$node_bin" --no-deprecation "${DIR}/_keyid.js" "$1"
-}
-
-JWT_CRT="${CERT_FILE}.crt"
-JWT_KEY="${CERT_FILE}.pem"
-JWT_KID="${CERT_FILE}.kid"
-
-if [ ! -f $JWT_CRT ] || [ ! -f $JWT_KEY ] || [ ! -f $JWT_KID ]; then
-  rm -f $JWT_CRT $JWT_KEY $JWT_KID
-  mkdir -p "${CERT_DIR}"
-  openssl ecparam -name prime256v1 -genkey -noout -out "${JWT_KEY}" 2>/dev/null
-  openssl req -x509 -new -nodes -days "${CRT_EXPIRY_DAYS}" -key "${JWT_KEY}" -subj "/CN=api.${CN}" -out "${JWT_CRT}" 2>/dev/null
-  openssl ec -in "${JWT_KEY}" -pubout -outform DER -out "${CERT_FILE}.der" 2>/dev/null
-  keyid "${CERT_FILE}.der" >"${JWT_KID}"
-  rm "${CERT_FILE}.der"
-fi

scripts/gen-vpn-certs

@@ -1,48 +0,0 @@
-#!/bin/bash -e
-
-usage() {
-  echo "usage: $0 COMMON_NAME [OUT]"
-  echo
-  echo "  COMMON_NAME   the domain name the certificate is valid for, eg. example.com"
-  echo "  OUT           path to output directory generated files will be placed in"
-  echo
-}
-
-if [ -z "$1" ]; then
-  usage
-  exit 1
-fi
-
-CMD="$(realpath "$0")"
-DIR="$(dirname "${CMD}")"
-
-CN="$1"
-OUT="$(realpath "${2:-.}")"
-
-# shellcheck source=scripts/ssl-common.sh
-source "${DIR}/ssl-common.sh"
-
-VPN_PKI="$(realpath "${OUT}/vpn")"
-VPN_CA="${VPN_PKI}/ca.crt"
-VPN_CRT="${VPN_PKI}/issued/vpn.${CN}.crt"
-VPN_KEY="${VPN_PKI}/private/vpn.${CN}.key"
-VPN_DH="${VPN_PKI}/dh.pem"
-
-if [ ! -f $VPN_CA ] || [ ! -f $VPN_CRT ] || [ ! -f $VPN_KEY ] || [ ! -f $VPN_DH ]; then
-
-  rm -f $VPN_CA $VPN_CRT $VPN_DH $VPN_KEY
-
-  # generate VPN CA
-  "$easyrsa_bin" --pki-dir="${VPN_PKI}" init-pki &>/dev/null
-  "$easyrsa_bin" --pki-dir="${VPN_PKI}" --days="${CA_EXPIRY_DAYS}" --req-cn="vpn-ca.${CN}" build-ca nopass 2>/dev/null
-
-  # generate and sign vpn server certificate
-  "$easyrsa_bin" --pki-dir="${VPN_PKI}" --days="${CRT_EXPIRY_DAYS}" build-server-full "vpn.${CN}" nopass 2>/dev/null
-
-  # generate vpn dhparams (keysize of 2048 will do, 4096 can wind up taking hours to generate)
-  "$easyrsa_bin" --pki-dir="${VPN_PKI}" --keysize=2048 gen-dh 2>/dev/null
-
-  # update indexes and generate CRLs
-  "$easyrsa_bin" --pki-dir="${VPN_PKI}" update-db 2>/dev/null
-  "$easyrsa_bin" --pki-dir="${VPN_PKI}" gen-crl 2>/dev/null
-fi

scripts/logger.sh

@@ -1,62 +0,0 @@
-#!/bin/sh
-
-BLACK=`tput setaf 0`
-RED=`tput setaf 1`
-GREEN=`tput setaf 2`
-YELLOW=`tput setaf 3`
-BLUE=`tput setaf 4`
-MAGENTA=`tput setaf 5`
-CYAN=`tput setaf 6`
-WHITE=`tput setaf 7`
-
-BOLD=`tput bold`
-RESET=`tput sgr0`
-
-log_raw () {
-    local COLOR="${WHITE}"
-    local LEVEL="${1}"
-    local MESSAGE="${2}"
-    case "${LEVEL}" in
-        info)
-            COLOR="${BLUE}"
-            ;;
-        warn)
-            COLOR="${YELLOW}"
-            ;;
-        fatal)
-            COLOR="${RED}"
-            ;;
-        *)
-            LEVEL="debug"
-            ;;
-    esac
-    LEVEL="${LEVEL}     "
-    echo "[$(date +%T)] ${COLOR}$(echo "${LEVEL:0:5}" | tr '[:lower:]' '[:upper:]')${RESET} ${MESSAGE}";
-}
-
-log () {
-    log_raw "debug" "${1}"
-}
-
-info () {
-    log_raw "info" "${1}";
-}
-
-warn () {
-    log_raw "warn" "${1}";
-}
-
-die () {
-    log_raw "fatal" "${1}";
-    exit 1;
-}
-
-die_unless_forced () {
-    if [ ! -z "$1" ]; then
-        log_raw "warn" "$2";
-        return;
-    fi
-    
-    log_raw "fatal" "$2";
-    die "Use -f to forcibly upgrade.";
-}

scripts/make-env

@@ -1,77 +0,0 @@
-#!/bin/bash -e
-
-usage() {
-  echo "usage: $0"
-  echo
-  echo "Required Variables:"
-  echo
-  echo "  DOMAIN"
-  echo "  ROOT_CA    Path to root CA certificate"
-  echo "  ROOT_CRT   Path to root/wildcard certificate"
-  echo "  ROOT_KEY   Path to root/wildcard private key"
-  echo "  JWT_CRT    Path to Token Auth certificate"
-  echo "  JWT_KEY    Path to Token Auth private key"
-  echo "  JWT_KID    Path to KeyID for the Token Auth certificate"
-  echo "  VPN_CA     Path to the VPN CA certificate"
-  echo "  VPN_CRT    Path to the VPN server certificate"
-  echo "  VPN_KEY    Path to the VPN server private key"
-  echo "  VPN_DH     Path to the VPN server Diffie Hellman parameters"
-  echo "  SUPERUSER_EMAIL     Email address of the superuser"
-  echo "  SUPERUSER_PASSWORD  Password of the superuser"
-  echo
-}
-
-for var in DOMAIN ROOT_CA ROOT_CRT ROOT_KEY JWT_CRT JWT_KEY JWT_KID VPN_CA VPN_CRT VPN_KEY VPN_DH SUPERUSER_EMAIL SUPERUSER_PASSWORD; do
-  if [ -z "${!var-}" ]; then
-    usage
-    exit 1
-  fi
-done
-
-randstr() {
-  LC_CTYPE=C tr -dc A-Za-z0-9 < /dev/urandom | fold -w "${1:-32}" | head -n 1
-}
-
-b64encode() {
-    echo "$@" | base64 --wrap=0 2>/dev/null || echo "$@" | base64 --break=0 2>/dev/null
-}
-
-b64file() {
-  b64encode "$(cat "$@")"
-}
-
-# buckets to create in the S3 service...
-REGISTRY2_S3_BUCKET="registry-data"
-
-cat <<STR
-export OPENBALENA_PRODUCTION_MODE=false
-export OPENBALENA_COOKIE_SESSION_SECRET=$(randstr 32)
-export OPENBALENA_HOST_NAME=$DOMAIN
-export OPENBALENA_JWT_SECRET=$(randstr 32)
-export OPENBALENA_REGISTRY2_S3_BUCKET=${REGISTRY2_S3_BUCKET}
-export OPENBALENA_RESINOS_REGISTRY_CODE=$(randstr 32)
-export OPENBALENA_ROOT_CA=$(b64file "${ROOT_CA}")
-export OPENBALENA_ROOT_CRT=$(b64file "${ROOT_CRT}")
-export OPENBALENA_ROOT_KEY=$(b64file "${ROOT_KEY}")
-export OPENBALENA_TOKEN_AUTH_BUILDER_TOKEN=$(randstr 64)
-export OPENBALENA_TOKEN_AUTH_PUB=$(b64file "$JWT_CRT")
-export OPENBALENA_TOKEN_AUTH_KEY=$(b64file "$JWT_KEY")
-export OPENBALENA_TOKEN_AUTH_KID=$(b64file "$JWT_KID")
-export OPENBALENA_VPN_CA=$(b64file "$VPN_CA")
-export OPENBALENA_VPN_CA_CHAIN=$(b64file "$VPN_CA")
-export OPENBALENA_VPN_SERVER_CRT=$(b64file "$VPN_CRT")
-export OPENBALENA_VPN_SERVER_KEY=$(b64file "$VPN_KEY")
-export OPENBALENA_VPN_SERVER_DH=$(b64file "$VPN_DH")
-export OPENBALENA_VPN_SERVICE_API_KEY=$(randstr 32)
-export OPENBALENA_API_VPN_SERVICE_API_KEY=$(randstr 32)
-export OPENBALENA_REGISTRY_SECRET_KEY=$(randstr 32)
-export OPENBALENA_S3_ACCESS_KEY=$(randstr 32)
-export OPENBALENA_S3_BUCKETS="${REGISTRY2_S3_BUCKET}"
-export OPENBALENA_S3_ENDPOINT="https://s3.${DOMAIN}"
-export OPENBALENA_S3_REGION=us-east-1
-export OPENBALENA_S3_SECRET_KEY=$(randstr 32)
-export OPENBALENA_SSH_AUTHORIZED_KEYS=
-export OPENBALENA_SUPERUSER_EMAIL=$SUPERUSER_EMAIL
-export OPENBALENA_SUPERUSER_PASSWORD=$(printf "%q" "${SUPERUSER_PASSWORD}")
-export OPENBALENA_ACME_CERT_ENABLED=${ACME_CERT_ENABLED:-false}
-STR

scripts/migrate-registry-storage

@@ -1,29 +0,0 @@
-#!/bin/sh
-
-migrate_data_to_s3 () {
-  BUCKET="${1:-registry-data}"
-
-  if [ -z "${BUCKET}" ]; then return 1; fi
-
-  if [ -n "${DOCKER_HOST}" ]; then
-      log "Using docker host: ${DOCKER_HOST}"
-      export DOCKER_HOST="${DOCKER_HOST}"
-  fi
-
-  REGISTRY_CONTAINER="$(docker ps | grep registry_ | awk '{print $1}')"
-  S3_CONTAINER="$(docker ps | grep s3_ | awk '{print $1}')"
-
-  if [ -z "${REGISTRY_CONTAINER}" ] || [ -z "${S3_CONTAINER}" ]; then return 2; fi
-
-  REGISTRY_VOLUME="$(docker inspect "${REGISTRY_CONTAINER}" | jq -r '.[].Mounts | map(select(.Destination=="/data")) | .[0].Source')"
-  S3_VOLUME=$(docker inspect "${S3_CONTAINER}" | jq -r '.[].Mounts | map(select(.Destination=="/export")) | .[0].Source')
-
-  if [ -z "${REGISTRY_VOLUME}" ] || [ -z "${S3_VOLUME}" ]; then return 3; fi
-
-  # run the S3 container image, and copy the data partition into S3...
-  docker run -it --rm \
-      -v "${REGISTRY_VOLUME}:/data" \
-      -v "${S3_VOLUME}:/s3" \
-      --name "migrate-registry" alpine \
-      sh -c "mkdir -p /s3/${BUCKET}/data && cp -r /data/docker /s3/${BUCKET}/data/"
-}

scripts/patch-hosts

@@ -1,30 +0,0 @@
-#!/bin/bash -e
-
-usage() {
-  echo "usage: $0 DOMAIN"
-  echo
-  echo "  DOMAIN  the domain name to add host entries for, eg. example.com"
-  echo
-}
-
-if [ -z "$1" ]; then
-  usage
-  exit 1
-fi
-
-SERVICES="api registry vpn db s3 redis"
-DOMAIN="$1"
-
-# We need sudo to write to /etc/hosts, so first write to a temp file and then
-# append all entries to hosts file.
-tmp="$(mktemp --tmpdir openbalena.XXXX)"
-for service in $SERVICES; do
-  name="${service}.${DOMAIN}"
-  if ! grep "\\s$name" /etc/hosts >/dev/null 2>&1 ; then
-    echo "adding $name"
-    echo "127.0.0.1 $name" >>"${tmp}"
-  fi
-done
-# shellcheck disable=SC2024
-sudo tee -a /etc/hosts >/dev/null <"${tmp}"
-rm -f "${tmp}"

scripts/quickstart

@@ -1,155 +0,0 @@
-#!/bin/bash -e
-
-BLACK=`tput setaf 0`
-RED=`tput setaf 1`
-GREEN=`tput setaf 2`
-YELLOW=`tput setaf 3`
-BLUE=`tput setaf 4`
-MAGENTA=`tput setaf 5`
-CYAN=`tput setaf 6`
-WHITE=`tput setaf 7`
-
-BOLD=`tput bold`
-RESET=`tput sgr0`
-
-# for macos machines, we need proper OpenSSL...
-OPENSSL_VERSION=$(openssl version -v)
-if [[ "${OPENSSL_VERSION}" =~ ^LibreSSL.*$ ]]; then
-  echo -e "${RED}ERROR: You may not have a compatible OpenSSL version (${OPENSSL_VERSION}). Please install OpenSSL version 1.0.2q or above.${RESET}"
-  if [ $(uname) == 'Darwin' ]; then
-        echo 'OpenSSL is required to build openBalena on macOS. To install with brew, run'
-        echo ''
-        echo '    brew install openssl'
-        echo ''
-    fi
-  exit 1
-fi
-
-source "${BASH_SOURCE%/*}/_realpath"
-
-domainResolves() {
-  getent hosts "$1" > /dev/null 2>&1
-}
-
-CMD="$(realpath "$0")"
-DIR="$(dirname "${CMD}")"
-BASE_DIR="$(dirname "${DIR}")"
-CONFIG_DIR="${BASE_DIR}/config"
-CERTS_DIR="${CONFIG_DIR}/certs"
-
-DOMAIN=openbalena.local
-
-usage() {
-  echo "usage: $0 [-c] [-h] [-p] [-d DOMAIN] -U EMAIL -P PASSWORD"
-  echo
-  echo "  -c           enable the ACME certificate service in staging or production mode."
-  echo "  -p           patch hosts - patch the host /etc/hosts file"
-  echo "  -d DOMAIN    the domain name this deployment will run as, eg. example.com. Default is 'openbalena.local'"
-  echo "  -U EMAIL     the email address of the superuser account, used to login to your install from the Balena CLI"
-  echo "  -P PASSWORD  the password to use for the superuser account."
-  echo
-}
-
-show_help=false
-patch_hosts=false
-while getopts ":chpxd:U:P:" opt; do
-  case "${opt}" in
-    h) show_help=true;;
-    p) patch_hosts=true;;
-    x) set -x;;
-    d) DOMAIN="${OPTARG}";;
-    U) SUPERUSER_EMAIL="${OPTARG}";;
-    P) SUPERUSER_PASSWORD="${OPTARG}";;
-    c) ACME_CERT_ENABLED="true";;
-    *)
-      echo "Invalid argument: -${OPTARG}"
-      usage
-      exit 1
-      ;;
-  esac
-done
-shift $((OPTIND-1))
-
-if [ -z "${SUPERUSER_EMAIL}" ] || [ -z "${SUPERUSER_PASSWORD}" ]; then
-  usage
-  exit 1
-fi
-
-if [ "$show_help" = "true" ]; then
-  usage
-  exit 1
-fi
-
-if [ ! -z "$ACME_CERT_ENABLED" ]; then
-  echo "${BLUE}[INFO]${RESET} ACME Certificate request is ${BOLD}ENABLED${RESET}."
-  
-  if ! domainResolves "api.${DOMAIN}"; then
-    echo "${YELLOW}[WARN]${RESET} Unable to resolve \"api.${DOMAIN}\"!"
-    echo "${YELLOW}[WARN]${RESET} This might mean that you cannot use an ACME issued certificate."
-  fi
-fi
-
-echo_bold() {
-  echo "${BOLD}${@}${RESET}"
-}
-
-echo_bold "==> Creating new configuration at: $CONFIG_DIR"
-mkdir -p "$CONFIG_DIR" "$CERTS_DIR"
-
-echo_bold "==> Bootstrapping easy-rsa..."
-source "${DIR}/ssl-common.sh"
-
-echo_bold "==> Generating root CA cert..."
-# shellcheck source=scripts/gen-root-ca
-source "${DIR}/gen-root-ca" "${DOMAIN}" "${CERTS_DIR}"
-
-echo_bold "==> Generating root cert chain for haproxy..."
-# shellcheck source=scripts/gen-root-cert
-source "${DIR}/gen-root-cert" "${DOMAIN}" "${CERTS_DIR}"
-
-echo_bold "==> Generating token auth cert..."
-# shellcheck source=scripts/gen-token-auth-cert
-source "${DIR}/gen-token-auth-cert" "${DOMAIN}" "${CERTS_DIR}"
-
-echo_bold "==> Generating VPN CA, cert and dhparam (this may take a while)..."
-# shellcheck source=scripts/gen-vpn-certs
-source "${DIR}/gen-vpn-certs" "${DOMAIN}" "${CERTS_DIR}"
-
-echo_bold "==> Setting up environment..."
-# shellcheck source=scripts/make-env
-cat >"${CONFIG_DIR}/activate" <(source "${DIR}/make-env")
-
-echo_bold "==> Adding default compose file..."
-cp "${BASE_DIR}/compose/template.yml" "${CONFIG_DIR}/docker-compose.yml"
-
-if [ "${patch_hosts}" = "true" ]; then
-  echo_bold "==> Patching /etc/hosts..."
-  # shellcheck source=scripts/patch-hosts
-  source "${DIR}/patch-hosts" "${DOMAIN}"
-fi
-
-echo_bold "==> Success!"
-echo '  - Start the instance with: ./scripts/compose up -d'
-echo '  - Stop the instance with: ./scripts/compose stop'
-echo '  - To create a single, flat, docker-compose.yml file, run:'
-echo ''
-echo '      ./scripts/compose config > docker-compose.yml'
-echo ''
-
-if [ -z "${ACME_CERT_ENABLED}" ]; then
-  echo "  - Use the following certificate with Balena CLI: ${CERTS_DIR}/root/ca.crt"
-
-  case $(uname) in
-    Darwin)
-      echo ''
-      printf '    On macOS:\n\n'
-      printf '      sudo security add-trusted-cert -d -r trustRoot -k "/Library/Keychains/System.keychain" "%s/root/ca.crt"\n' "${CERTS_DIR}"
-      echo ''
-      ;;
-    *)
-      ;;
-  esac
-
-  echo -e "    ${YELLOW}IMPORTANT:${RESET} You will need to restart your Docker daemon after trusting this certificate to allow your workstation to push images to the registry."
-  echo ''
-fi

scripts/ssl-common.sh

@@ -1,25 +0,0 @@
-#!/bin/bash -e
-# shellcheck disable=SC2034
-
-# ensure we have `easyrsa` available
-if [ -z "${easyrsa_bin-}" ] || [ ! -x "${easyrsa_bin}" ]; then
-    easyrsa_bin="$(command -v easyrsa 2>/dev/null || true)"
-    if [ -z "${easyrsa_bin}" ]; then
-        easyrsa_dir="$(mktemp -dt easyrsa.XXXXXXXX)"
-        easyrsa_url="https://github.com/OpenVPN/easy-rsa/releases/download/v3.0.5/EasyRSA-nix-3.0.5.tgz"
-        echo "  - Downloading easy-rsa..."
-        (cd "${easyrsa_dir}"; curl -sL "${easyrsa_url}" | tar xz --strip-components=1)
-        easyrsa_bin="${easyrsa_dir}/easyrsa"
-        # shellcheck disable=SC2064
-        trap "rm -rf \"${easyrsa_dir}\"" EXIT
-    fi
-    export EASYRSA_BATCH=1
-    export EASYRSA_KEY_SIZE=4096
-fi
-
-# setup ROOT_PKI path
-ROOT_PKI="$(realpath "${OUT}/root")"
-
-# global expiry settings
-CA_EXPIRY_DAYS=3650
-CRT_EXPIRY_DAYS=730

scripts/upgrade-1.x-to-2.0

@@ -1,78 +0,0 @@
-#!/bin/sh
-
-source "${BASH_SOURCE%/*}/logger.sh"
-source "${BASH_SOURCE%/*}/migrate-registry-storage"
-
-# This script takes a v1.x.x install and updates the compose stack to use S3 as your
-# registry storage.
-
-source "${BASH_SOURCE%/*}/_realpath"
-
-DIR="$(dirname $(realpath "$0"))"
-BASE_DIR="$(dirname "${DIR}")"
-CONFIG_DIR="${BASE_DIR}/config"
-CONFIG_FILE="${CONFIG_DIR}/activate"
-
-# Step 1. Make sure a config exists...
-[ -f "${CONFIG_FILE}" ] || die "Unable to find existing config!";
-
-info "Preparing to upgrade..."
-source "${CONFIG_FILE}"
-
-while getopts "f" opt; do
-  case "${opt}" in
-    f) 
-        warn "Forcing upgrade! I hope you know what you're doing..."
-        FORCE_UPGRADE=1
-        ;;
-    *)
-      echo "Invalid argument: ${OPTARG}"
-      exit 1
-      ;;
-  esac
-done
-shift $((OPTIND-1))
-
-# Step 2. Check if the S3 configuration already exists...
-upgrade_required () {
-    [ -z "${OPENBALENA_REGISTRY2_S3_BUCKET}" ] || return 1;
-    [ -z "${OPENBALENA_S3_ACCESS_KEY}" ] || return 1;
-    [ -z "${OPENBALENA_S3_ENDPOINT}" ] || return 1;
-    [ -z "${OPENBALENA_S3_REGION}" ] || return 1;
-    [ -z "${OPENBALENA_S3_SECRET_KEY}" ] || return 1;
-}
-upgrade_required || die_unless_forced "${FORCE_UPGRADE}" "Configuration may already be using S3 for Registry storage!"
-
-# Step 3. Create missing S3 configuration...
-randstr() {
-  LC_CTYPE=C tr -dc A-Za-z0-9 < /dev/urandom | fold -w "${1:-32}" | head -n 1
-}
-
-upsert_config () {
-    var="${1}"
-    value="${2}"
-
-    if [ -z "${!var}" ]; then
-        echo "export ${1}=${2}" >> "${CONFIG_FILE}"
-    else
-        sed -i '' "s~export ${1}=.*~export ${1}=${2}~" "${CONFIG_FILE}"
-    fi
-}
-
-upsert_config "OPENBALENA_REGISTRY2_S3_BUCKET" "registry-data" || warn "Failed to update config value OPENBALENA_REGISTRY2_S3_BUCKET"
-upsert_config "OPENBALENA_S3_ACCESS_KEY" "$(randstr 32)" || warn "Failed to update config value OPENBALENA_S3_ACCESS_KEY"
-upsert_config "OPENBALENA_S3_ENDPOINT" "https://s3.${OPENBALENA_HOST_NAME}" || warn "Failed to update config value OPENBALENA_S3_ENDPOINT"
-upsert_config "OPENBALENA_S3_REGION" "us-east-1" || warn "Failed to update config value OPENBALENA_S3_REGION"
-upsert_config "OPENBALENA_S3_SECRET_KEY" "$(randstr 32)" || warn "Failed to update config value OPENBALENA_S3_SECRET_KEY"
-
-# Step 4. Migrate Registry data to S3...
-info "Copying data from the Registry volume to the S3 volume..."
-migrate_data_to_s3 "registry-data"
-case $? in
-    1)  die "Invalid bucket name";;
-    2)  die "Unable to find the running Registry or S3 containers";;
-    3)  die "Unable to determine the data volumes for the Registry or S3 containers";;
-    *)  info "Registry data copied"
-        ;;
-esac
-info "Upgrade complete"

src/README

@@ -1 +0,0 @@
-This is the working folder for any specific container you might want to work on.

src/balena-tests/Dockerfile

@@ -0,0 +1,34 @@
+FROM ubuntu:24.04
+
+# renovate: datasource=github-releases depName=balena-io/balena-cli
+ARG BALENA_CLI_VERSION=v18.2.20
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+	bash \
+	ca-certificates \
+	curl \
+	jq \
+	openssl \
+	procmail \
+	qemu-utils \
+	unzip \
+	wget \
+    && rm -rf /var/lib/apt/lists/*
+
+# https://docs.docker.com/engine/install/ubuntu/#install-using-the-convenience-script
+RUN curl -fsSL https://get.docker.com | sh
+
+WORKDIR /opt
+
+RUN set -x; arch=$(uname -m | sed 's/86_64/64/g') \
+    && wget -q "https://github.com/balena-io/balena-cli/releases/download/${BALENA_CLI_VERSION}/balena-cli-${BALENA_CLI_VERSION}-linux-${arch}-standalone.zip" \
+    && unzip -q "balena-cli-${BALENA_CLI_VERSION}-linux-${arch}-standalone.zip" \
+    && rm -rf "balena-cli-${BALENA_CLI_VERSION}-linux-${arch}-standalone.zip"
+
+ENV PATH=/opt/balena-cli:${PATH}
+
+COPY functions balena.sh /usr/sbin/
+
+WORKDIR /balena
+
+CMD /usr/sbin/balena.sh

src/balena-tests/balena.sh

@@ -0,0 +1,328 @@
+#!/usr/bin/env bash
+
+# shellcheck disable=SC2154,SC2034,SC1090
+set -ae
+
+curl_opts="--retry 3 --fail"
+if [[ $VERBOSE =~ on|On|Yes|yes|true|True ]]; then
+    set -x
+    curl_opts="${curl_opts} --verbose"
+else
+    curl_opts="${curl_opts} --silent"
+fi
+
+source /usr/sbin/functions
+
+function remove_test_assets() {
+    rm -rf /balena/config.json \
+      "${GUEST_IMAGE}" \
+      "${GUEST_IMAGE%.*}.ready" \
+      "${tmpbuild}" \
+      /tmp/*.img
+}
+
+function remove_update_lock() {
+    rm -f /tmp/balena/updates.lock
+}
+
+function cleanup() {
+   shutdown_dut
+   remove_test_assets
+   remove_update_lock
+
+   # crash loop backoff
+   sleep "$(( (RANDOM % 5) + 5 ))s"
+}
+trap 'cleanup' EXIT
+
+function shutdown_dut() {
+    local balena_device_uuid
+    balena_device_uuid="$(cat </balena/config.json | jq -r .uuid)"
+
+    if [[ -n $balena_device_uuid ]]; then
+        with_backoff balena device "${balena_device_uuid}"
+        with_backoff balena device shutdown -f "${balena_device_uuid}"
+    fi
+}
+
+function set_update_lock {
+    if [[ -n "$BALENA_SUPERVISOR_ADDRESS" ]] && [[ -n "$BALENA_SUPERVISOR_API_KEY" ]]; then
+        while [[ $(curl ${curl_opts} "${BALENA_SUPERVISOR_ADDRESS}/v1/device?apikey=${BALENA_SUPERVISOR_API_KEY}" \
+          -H "Content-Type: application/json" | jq -r '.update_pending') == 'true' ]]; do
+
+            curl ${curl_opts} "${BALENA_SUPERVISOR_ADDRESS}/v1/device?apikey=${BALENA_SUPERVISOR_API_KEY}" \
+              -H "Content-Type: application/json" | jq -r
+
+            sleep "$(( (RANDOM % 3) + 3 ))s"
+        done
+        sleep "$(( (RANDOM % 5) + 5 ))s"
+
+        # https://www.balena.io/docs/learn/deploy/release-strategy/update-locking/
+        lockfile /tmp/balena/updates.lock
+    fi
+}
+
+function update_ca_certificates() {
+    # only set CA bundle if using private certificate chain
+    if [[ -e "${CERTS}/ca-bundle.pem" ]]; then
+        if [[ "$(readlink -f "${CERTS}/${TLD}-chain.pem")" =~ \/private\/ ]]; then
+            mkdir -p /usr/local/share/ca-certificates
+            cat <"${CERTS}/ca-bundle.pem" > /usr/local/share/ca-certificates/balenaRootCA.crt
+            # shellcheck disable=SC2034
+            CURL_CA_BUNDLE=${CURL_CA_BUNDLE:-${CERTS}/ca-bundle.pem}
+            NODE_EXTRA_CA_CERTS=${NODE_EXTRA_CA_CERTS:-${CURL_CA_BUNDLE}}
+            # (TBC) refactor to use NODE_EXTRA_CA_CERTS instead of ROOT_CA
+            # https://github.com/balena-io/e2e/blob/master/conf.js#L12-L14
+            # https://github.com/balena-io/e2e/blob/master/Dockerfile#L82-L83
+            # ... or
+            # https://thomas-leister.de/en/how-to-import-ca-root-certificate/
+            # https://github.com/puppeteer/puppeteer/issues/2377
+            ROOT_CA=${ROOT_CA:-$(cat <"${NODE_EXTRA_CA_CERTS}" | openssl base64 -A)}
+        else
+            rm -f /usr/local/share/ca-certificates/balenaRootCA.crt
+            unset NODE_EXTRA_CA_CERTS CURL_CA_BUNDLE ROOT_CA
+        fi
+        update-ca-certificates
+    fi
+}
+
+function wait_for_api() {
+    while ! curl ${curl_opts} "https://api.${DNS_TLD}/ping"; do
+        sleep "$(( (RANDOM % 5) + 5 ))s"
+    done
+}
+
+function open_balena_login() {
+    while ! balena login --credentials \
+      --email "${SUPERUSER_EMAIL}" \
+      --password "${SUPERUSER_PASSWORD}"; do
+        sleep "$(( (RANDOM % 5) + 5 ))s"
+    done
+}
+
+function create_fleet() {
+    if ! balena fleet "${TEST_FLEET}"; then
+        # wait for API to load DT contracts
+        while ! balena fleet create "${TEST_FLEET}" --type "${DEVICE_TYPE}"; do
+            sleep "$(( (RANDOM % 5) + 5 ))s"
+        done
+
+        # FIXME: on openBalena 'balena devices supported' always returns empty list
+        balena devices supported
+    fi
+}
+
+function download_os_image() {
+    if ! [[ -s "$GUEST_IMAGE" ]]; then
+        with_backoff wget -qO /tmp/balena.zip \
+          "${BALENA_API_URL}/download?deviceType=${DEVICE_TYPE}&version=${OS_VERSION:1}&fileType=.zip"
+
+        unzip -oq /tmp/balena.zip -d /tmp
+
+        cat <"$(find /tmp/ -type f -name '*.img' | head -n 1)" >"${GUEST_IMAGE}"
+
+        rm /tmp/balena.zip
+    fi
+}
+
+function configure_virtual_device() {
+    while ! [[ -s "$GUEST_IMAGE" ]]; do sleep "$(( (RANDOM % 5) + 5 ))s"; done
+
+    if ! [[ -s /balena/config.json ]]; then
+        balena_device_uuid="$(openssl rand -hex 16)"
+
+        with_backoff balena device register "${TEST_FLEET}" \
+          --uuid "${balena_device_uuid}"
+
+        with_backoff balena config generate \
+          --version "${OS_VERSION:1}" \
+          --device "${balena_device_uuid}" \
+          --network ethernet \
+          --appUpdatePollInterval 10 \
+          --dev \
+          --output /balena/config.json
+    fi
+    cat </balena/config.json | jq -re
+
+    with_backoff balena os configure "${GUEST_IMAGE}" \
+      --fleet "${TEST_FLEET}" \
+      --version "${OS_VERSION#v}" \
+      --config-network ethernet \
+      --config /balena/config.json
+
+    touch "${GUEST_IMAGE%.*}.ready"
+}
+
+function check_device_status() {
+    if [[ -e /balena/config.json ]]; then
+        balena_device_uuid="$(cat </balena/config.json | jq -r .uuid)"
+
+        if [[ -n $balena_device_uuid ]]; then
+            is_online="$(balena devices --json --fleet "${TEST_FLEET}" \
+              | jq -r --arg uuid "${balena_device_uuid}" '.[] | select(.uuid==$uuid).is_online == true')"
+
+            if [[ $is_online =~ true ]]; then
+                return 0
+            else
+                return 1
+            fi
+        fi
+    fi
+}
+
+function wait_for_device() {
+    while ! check_device_status; do sleep "$(( (RANDOM % 5) + 5 ))s"; done
+}
+
+function registry_auth() {
+    if [[ -n $REGISTRY_USER ]] && [[ -n $REGISTRY_PASS ]]; then
+        with_backoff docker login -u "${REGISTRY_USER}" -p "${REGISTRY_PASS}"
+
+        printf '{"https://index.docker.io/v1/": {"username":"%s", "password":"$s"}}' \
+          "${REGISTRY_USER}" "${REGISTRY_PASS}" | jq -r > ~/.balena/secrets.json
+    fi
+}
+
+function deploy_release() {
+    tmpbuild="$(mktemp -d)"
+    pushd "${tmpbuild}"
+
+    echo 'FROM hello-world' >Dockerfile
+
+    while ! balena deploy \
+      --ca "${DOCKER_CERT_PATH}/ca.pem" \
+      --cert "${DOCKER_CERT_PATH}/cert.pem" \
+      --key "${DOCKER_CERT_PATH}/key.pem" \
+      "${TEST_FLEET}"; do
+
+        sleep "$(( (RANDOM % 5) + 5 ))s"
+    done
+    popd
+}
+
+function get_releases() {
+      with_backoff balena releases --json "${TEST_FLEET}"
+}
+
+function get_release_commit() {
+      echo "$(get_releases)" | jq -re \
+        'select((.[].status=="success")
+        and (.[].is_invalidated==false)
+        and (.[].is_final==true)
+        and (.[].release_type=="final"))[0].commit'
+}
+
+function get_release_id() {
+      echo "$(get_releases)" | jq -re \
+        'select((.[].status=="success")
+        and (.[].is_invalidated==false)
+        and (.[].is_final==true)
+        and (.[].release_type=="final"))[0].id'
+}
+
+function supervisor_update_target_state() {
+    local balena_device_uuid
+    balena_device_uuid="$(cat </balena/config.json | jq -r .uuid)"
+
+    if [[ -n $balena_device_uuid ]]; then
+        while ! curl ${curl_opts} "https://api.${DNS_TLD}/supervisor/v1/update" \
+          --header "Content-Type: application/json" \
+          --header "Authorization: Bearer $(cat <~/.balena/token)" \
+          --data "{\"uuid\": \"${balena_device_uuid}\", \"data\": {\"force\": true}}"; do
+
+            sleep "$(( (RANDOM % 5) + 5 ))s"
+        done
+    fi
+}
+
+function check_running_release() {
+    local balena_device_uuid
+    balena_device_uuid="$(cat </balena/config.json | jq -r .uuid)"
+
+    local should_be_running_release
+    should_be_running_release="$(get_release_commit)"
+    [[ -z $should_be_running_release ]] && false
+
+    if [[ -n $balena_device_uuid ]]; then
+        while ! [[ $(balena device "${balena_device_uuid}" | grep -E ^COMMIT | awk '{print $2}') =~ ${should_be_running_release} ]]; do
+            running_release_id="$(balena device "${balena_device_uuid}" | grep -E ^COMMIT | awk '{print $2}')"
+            printf 'please wait, device %s should be running %s, but is still running %s...\n' \
+              "${balena_device_uuid}" \
+              "${should_be_running_release}" \
+              "${running_release_id}"
+
+            sleep "$(( (RANDOM % 5) + 5 ))s"
+        done
+    fi
+}
+
+function get_os_version() {
+    local BALENARC_BALENA_URL
+    BALENARC_BALENA_URL="$(echo "${BALENA_API_URL}" | sed 's#https://api\.##g')"
+
+    local os_version
+    os_version=${OS_VERSION:-$(with_backoff balena os versions "${DEVICE_TYPE}" | head -n 1)}
+    echo "${os_version}"
+}
+
+function upload_release_asset() {
+    if [[ "$RELEASE_ASSETS_TEST" =~ true ]]; then
+        local release_id
+        release_id=${1:-1}
+        release_asset="$(find / -type f -name '*.png' | head -n 1)"
+
+        curl ${curl_opts} "https://api.${DNS_TLD}/resin/release_asset" \
+          --header "Authorization: Bearer $(cat <~/.balena/token)" \
+          --form "asset=@${release_asset}" \
+          --form "release=${release_id}" \
+          --form "asset_key=$((RANDOM))-$(basename ${release_asset})" \
+          | jq -re .asset.href \
+          | xargs curl ${curl_opts} -o "/tmp/$((RANDOM))-$(basename ${release_asset})"
+    fi
+}
+
+# --- main
+if [[ "$PRODUCTION_MODE" =~ true ]]; then
+    exit
+fi
+
+if [[ -n "${BALENA_DEVICE_UUID}" ]]; then
+    # prepend the device UUID if running on balenaOS
+    TLD="${BALENA_DEVICE_UUID}.${DNS_TLD}"
+else
+    TLD="${DNS_TLD}"
+fi
+
+BALENA_API_URL=${BALENA_API_URL:-https://api.balena-cloud.com}
+BALENARC_BALENA_URL="${DNS_TLD}"
+CERTS=${CERTS:-/certs}
+CONF=${CONF:-/balena/${TLD}.env}
+DEVICE_TYPE=${DEVICE_TYPE:-generic-amd64}
+GUEST_DISK_SIZE=${GUEST_DISK_SIZE:-8}
+GUEST_IMAGE=${GUEST_IMAGE:-/balena/balena.img}
+OS_VERSION="$(get_os_version)"
+TEST_FLEET=${TEST_FLEET:-test-fleet}
+
+[[ -f "$CONF" ]] && source "${CONF}"
+
+update_ca_certificates  # ensure self-signed root CA certificate(s) trust
+
+registry_auth  # optionally authenticate with DockerHub (rate-limiting)
+
+wait_for_api  # spin here until the API is responding
+
+balena whoami || open_balena_login  # spin here until authenticated
+
+create_fleet  # spin here until the fleet is created
+
+# critical section
+set_update_lock
+download_os_image
+configure_virtual_device
+deploy_release
+upload_release_asset "$(get_release_id)"  # upload an additional asset to a release
+remove_update_lock
+# .. end
+
+wait_for_device  # spin here until test-device comes online
+check_running_release  # .. and ensure the device is running our release

src/balena-tests/functions

@@ -0,0 +1,32 @@
+# https://coderwall.com/p/--eiqg/exponential-backoff-in-bash
+function with_backoff() {
+    local max_attempts=${ATTEMPTS-5}
+    local timeout=${TIMEOUT-1}
+    local attempt=0
+    local exitCode=0
+
+    set +e
+    while [[ $attempt < $max_attempts ]]
+    do
+        "$@"
+        exitCode=$?
+
+        if [[ $exitCode == 0 ]]
+        then
+          break
+        fi
+
+        echo "Failure! Retrying in $timeout.." 1>&2
+        sleep "$timeout"
+        attempt=$(( attempt + 1 ))
+        timeout=$(( timeout * 2 ))
+    done
+
+    if [[ $exitCode != 0 ]]
+    then
+        echo "You've failed me for the last time! ($*)" 1>&2
+    fi
+
+    set -e
+    return $exitCode
+}

src/cert-manager/Dockerfile

@@ -0,0 +1,4 @@
+# https://github.com/balena-io/cert-manager
+FROM balena/cert-manager:v0.2.2
+
+COPY *.json /opt/

src/cert-manager/certs.json

@@ -0,0 +1,63 @@
+[
+  {
+    "request": {
+      "key": {
+        "algo": "${key_algo}",
+        "size": ${key_size}
+      },
+      "hosts": ${hosts},
+      "names": [
+        {
+          "C": "${country}",
+          "L": "${locality_name}",
+          "O": "${org}",
+          "OU": "${org_unit}",
+          "ST": "${state}"
+        }
+      ],
+      "CN": "${TLD}"
+    }
+  },
+  {
+    "request": {
+      "key": {
+        "algo": "${key_algo}",
+        "size": ${key_size}
+      },
+      "hosts": [
+        "vpn.${TLD}"
+      ],
+      "names": [
+        {
+          "C": "${country}",
+          "L": "${locality_name}",
+          "O": "${org}",
+          "OU": "${org_unit}",
+          "ST": "${state}"
+        }
+      ],
+      "CN": "vpn.${TLD}"
+    }
+  },
+  {
+	"request": {
+	  "key": {
+		"algo": "${key_algo}",
+		"size": ${key_size}
+	  },
+	  "hosts": [
+		"api.${TLD}"
+	  ],
+	  "names": [
+		{
+		  "C": "${country}",
+		  "L": "${locality_name}",
+		  "O": "${org}",
+		  "OU": "${org_unit}",
+		  "ST": "${state}"
+		}
+	  ],
+	  "CN": "api.${TLD}"
+	}
+  }
+]

src/cert-manager/keys.json

@@ -0,0 +1 @@
+[]

src/cert-provider/Dockerfile

@@ -1,22 +0,0 @@
-FROM alpine
-
-EXPOSE 80
-WORKDIR /usr/src/app
-VOLUME [ "/usr/src/app/certs" ]
-
-RUN apk add --update bash curl git openssl ncurses socat
-
-# from https://github.com/Neilpang/acme.sh/releases/tag/3.0.1
-RUN git clone https://github.com/Neilpang/acme.sh.git && \
-    cd acme.sh && \
-    git fetch && git fetch --tags && \
-    git checkout 3.0.1 . && \
-    ./acme.sh --install  \
-    --cert-home /usr/src/app/certs
-
-COPY entry.sh /entry.sh
-COPY cert-provider.sh ./cert-provider.sh
-COPY fake-le-bundle.pem ./
-
-ENTRYPOINT [ "/entry.sh" ]
-CMD [ "/usr/src/app/cert-provider.sh" ]

src/cert-provider/cert-provider.sh

@@ -1,190 +0,0 @@
-#!/usr/bin/env bash
-
-# the acme.sh client script, installed via Git in the Dockerfile...
-ACME_BIN="$(realpath ~/.acme.sh/acme.sh)"
-
-# the path to a bundle of certs to verify a LetsEncrypt staging certificate until Apr 2036...
-ACME_STAGING_CA="/usr/src/app/fake-le-bundle.pem"
-
-# the path to a file which stores the last successful mode of certificate we acquired...
-ACME_MODE_FILE="/usr/src/app/certs/last_run_mode"
-
-# colour output helpers...
-reset=$(tput -T xterm sgr0)
-red=$(tput -T xterm setaf 1)
-green=$(tput -T xterm setaf 2)
-yellow=$(tput -T xterm setaf 3)
-blue=$(tput -T xterm setaf 4)
-
-logError() {
-    echo "${red}[Error]${reset} $1"
-}
-
-logWarn() {
-    echo "${yellow}[Warn]${reset} $1"
-}
-
-logInfo() {
-    echo "${blue}[Info]${reset} $1"
-}
-
-logSuccess() {
-    echo "${green}[Success]${reset} $1"
-}
-
-logErrorAndStop() {
-    logError "$1 [Stopping]"
-    while true; do
-        # do nothing forever...
-        sleep 60
-    done
-}
-
-retryWithDelay() {
-    RETRIES=${2:-3}
-    DELAY=${3:-5}
-
-    local ATTEMPT=0
-    while [ "$RETRIES" -gt "$ATTEMPT" ]; do
-        (( ATTEMPT++ ))
-        logInfo "($ATTEMPT/$RETRIES) Connecting..."
-        if $1; then
-            logInfo "($ATTEMPT/$RETRIES) Success!"
-            return $?
-        fi
-
-        if [ "$RETRIES" -gt "$ATTEMPT" ]; then
-            logInfo "($ATTEMPT/$RETRIES) Failed. Retrying in ${DELAY} seconds..."
-            sleep "$DELAY"
-        else
-            logInfo "($ATTEMPT/$RETRIES) Failed!"
-        fi
-    done
-
-    return 1
-}
-
-waitForOnline() {
-    ADDRESS="${1,,}"
-
-    logInfo "Waiting for ${ADDRESS} to be available via HTTP..."
-    retryWithDelay "curl --output /dev/null --silent --head --fail --max-time 5 http://${ADDRESS}"
-}
-
-isUsingStagingCert() {
-    HOST="${1,,}"
-    echo "" | openssl s_client -host "$HOST" -port 443 -showcerts 2>/dev/null | awk '/BEGIN CERT/ {p=1} ; p==1; /END CERT/ {p=0}' | openssl verify -CAfile "$ACME_STAGING_CA" > /dev/null 2>&1
-}
-
-pre-flight() {
-    case "$ACTIVE" in
-        "true"|"yes")
-            ;;
-        *)
-            logError "ACTIVE variable is not enabled. Value should be \"true\" or \"yes\" to continue."
-            return 1
-            ;;
-    esac
-
-    if [ -z "$DOMAINS" ]; then
-        logError "DOMAINS must be set. Value should be a comma-delimited string of domains."
-        return 1
-    else
-        IFS=, read -r -a ACME_DOMAINS <<< "$DOMAINS"
-        IFS=' ' read -r -a ACME_DOMAIN_ARGS <<< "${ACME_DOMAINS[@]/#/-d }"
-    fi
-
-    if [ -z "$VALIDATION" ]; then
-        logInfo "VALIDATION not set. Using default: http-01"
-        VALIDATION="http-01"
-    else
-        case "$VALIDATION" in
-            "http-01")
-                logInfo "Using validation method: $VALIDATION"
-                ;;
-            *)
-                logError "VALIDATION is invalid. Use a valid value: http-01"
-                return 1
-                ;;
-        esac
-    fi
-
-    if [ -z "$OUTPUT_PEM" ]; then
-        logError "OUTPUT_PEM must be set. Value should be the path to install your certificate to."
-        return 1
-    fi
-}
-
-waitToSeeStagingCert() {
-    logInfo "Waiting for ${ACME_DOMAINS[0]} to use a staging certificate..."
-    retryWithDelay "isUsingStagingCert ${ACME_DOMAINS[0]}" 3 5
-}
-
-lastAcquiredCertFor() {
-    ACME_MODE="${1:-none}"
-    ACME_LAST_MODE="$(cat $ACME_MODE_FILE || echo '')"
-    logInfo "Last acquired certificate for ${ACME_LAST_MODE^^}"
-    [ "${ACME_LAST_MODE,,}" == "${ACME_MODE,,}" ]
-}
-
-acquireCertificate() {
-    ACME_MODE="${1:-staging}"
-    ACME_FORCE="${2:-false}"
-    ACME_OPTS=()
-
-    if [ "${ACME_FORCE,,}" == "true" ];then ACME_OPTS+=("--force"); fi
-    case "$ACME_MODE" in
-        "production")
-            logInfo "Using PRODUCTION mode"
-            ;;
-        *)
-            logInfo "Using STAGING mode"
-            ACME_OPTS+=("--staging")
-            ;;
-    esac
-
-    case "$VALIDATION" in
-        "http-01")
-            ACME_OPTS+=("--standalone")
-            ;;
-        *)
-            logError "VALIDATION is invalid. Use a valid value: http-01"
-            return 1
-            ;;
-    esac
-
-    if ! waitForOnline "${ACME_DOMAINS[0]}"; then
-        logError "Unable to access site over HTTP"
-        return 1
-    fi
-
-    logInfo "Issuing certificates..."
-    "$ACME_BIN" --server letsencrypt --issue "${ACME_OPTS[@]}" "${ACME_DOMAIN_ARGS[@]}"
-
-    logInfo "Installing certificates..." && \
-    "$ACME_BIN" --install-cert "${ACME_DOMAIN_ARGS[@]}" \
-    --cert-file      /tmp/cert.pem  \
-    --key-file       /tmp/key.pem  \
-    --fullchain-file /tmp/fullchain.pem \
-    --reloadcmd     "cat /tmp/fullchain.pem /tmp/key.pem > $OUTPUT_PEM" && \
-
-    echo "${ACME_MODE}" > "${ACME_MODE_FILE}"
-}
-
-pre-flight || logErrorAndStop "Unable to continue due to misconfiguration. See errors above."
-
-while ! waitForOnline "${ACME_DOMAINS[0]}"; do
-    logInfo "Unable to access ${ACME_DOMAINS[0]} on port 80. This is needed for certificate validation. Retrying in 30 seconds..."
-    sleep 30
-done
-
-if ! lastAcquiredCertFor "production"; then
-    acquireCertificate "staging" || logErrorAndStop "Unable to acquire a staging certificate."
-    waitToSeeStagingCert || logErrorAndStop "Unable to detect certificate change over. Cannot issue a production certificate."
-    acquireCertificate "production" "true" || logErrorAndStop "Unable to acquire a production certificate."
-fi
-
-logSuccess "Done!"
-
-logInfo "Running cron..."
-crond -f -d 7

src/cert-provider/entry.sh

@@ -1,3 +0,0 @@
-#!/bin/bash
-
-exec "$@"

src/cert-provider/fake-le-bundle.pem

@@ -1,119 +0,0 @@
------BEGIN CERTIFICATE-----
-MIIDrzCCApegAwIBAgIRALqMZiRNaRF4EGZS9urlj+0wDQYJKoZIhvcNAQELBQAw
-cTELMAkGA1UEBhMCVVMxMzAxBgNVBAoTKihTVEFHSU5HKSBJbnRlcm5ldCBTZWN1
-cml0eSBSZXNlYXJjaCBHcm91cDEtMCsGA1UEAxMkKFNUQUdJTkcpIERvY3RvcmVk
-IER1cmlhbiBSb290IENBIFgzMB4XDTAwMDkzMDIxMTIxOVoXDTIxMDEzMDE0MDEx
-NVowcTELMAkGA1UEBhMCVVMxMzAxBgNVBAoTKihTVEFHSU5HKSBJbnRlcm5ldCBT
-ZWN1cml0eSBSZXNlYXJjaCBHcm91cDEtMCsGA1UEAxMkKFNUQUdJTkcpIERvY3Rv
-cmVkIER1cmlhbiBSb290IENBIFgzMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB
-CgKCAQEAqUZjoRbjgXecPWxXkGCUEXcNrupL7dkbwc0jUTLFEDvcyfD1gYekY5uL
-D19uzYTl0pKZzzDXHJPnJY5EEp27nACFOm8XzX9sORAangP0OnGUkXJZDHM+8cX2
-EHJbfj0lg1JirRF3w2u1/KRuFEvIlWg3FdXdsSFHBF5z1Ij7MLn7Ska5c/5fKsDW
-EYzOMB6EBW1T9RDkVk/Q965EwDT4bR6BOXakasgfKrH9m1f6l9MmA0VnXdw9rZ+s
-TvMHG1yWBqNMSqCKe3jG6caWgN7llEbj5YsCWs32bz2dMftGkXBPcy1fNWvpeT7G
-Dz2Z0QWTlHkyXA2kGw32fdoXLHWOEwIDAQABo0IwQDAOBgNVHQ8BAf8EBAMCAQYw
-DwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUCFfaiceiU3kMT93gkI90uuInc0Qw
-DQYJKoZIhvcNAQELBQADggEBAF7lEtHuSN4j+xFQsM/ujaVKcn57VbrbTecnspmJ
-JA7Hrn6OErshGNO0p1/u14c7tGHKjtF1tEFFSVhbNXlKw9O99AfhmlFgdGcJKEHn
-ZctBB8bhNO387vbiCYIHdU/nSba9MCDYw2/UCtobZ6ao+KJA3IKmPixctAbn2Ikr
-EN9X0SXNP1gnqQP4VhZJIh6cd7rg9MimzoLlMI3m2z11dSGYbh8OWSdvA7aLbSGo
-gDO5H4WD8fgqEG0reSBO89eeH+we+BZxQtBiU3b9VMV0drc+7zC2NbXqeQwu6QTl
-fbJ8ytqcqUy0g5XSE6WCzPOL3H9r0j9G64dfotGlBA5tG6w=
------END CERTIFICATE-----
------BEGIN CERTIFICATE-----
-MIIFmDCCA4CgAwIBAgIQU9C87nMpOIFKYpfvOHFHFDANBgkqhkiG9w0BAQsFADBm
-MQswCQYDVQQGEwJVUzEzMDEGA1UEChMqKFNUQUdJTkcpIEludGVybmV0IFNlY3Vy
-aXR5IFJlc2VhcmNoIEdyb3VwMSIwIAYDVQQDExkoU1RBR0lORykgUHJldGVuZCBQ
-ZWFyIFgxMB4XDTE1MDYwNDExMDQzOFoXDTM1MDYwNDExMDQzOFowZjELMAkGA1UE
-BhMCVVMxMzAxBgNVBAoTKihTVEFHSU5HKSBJbnRlcm5ldCBTZWN1cml0eSBSZXNl
-YXJjaCBHcm91cDEiMCAGA1UEAxMZKFNUQUdJTkcpIFByZXRlbmQgUGVhciBYMTCC
-AiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBALbagEdDTa1QgGBWSYkyMhsc
-ZXENOBaVRTMX1hceJENgsL0Ma49D3MilI4KS38mtkmdF6cPWnL++fgehT0FbRHZg
-jOEr8UAN4jH6omjrbTD++VZneTsMVaGamQmDdFl5g1gYaigkkmx8OiCO68a4QXg4
-wSyn6iDipKP8utsE+x1E28SA75HOYqpdrk4HGxuULvlr03wZGTIf/oRt2/c+dYmD
-oaJhge+GOrLAEQByO7+8+vzOwpNAPEx6LW+crEEZ7eBXih6VP19sTGy3yfqK5tPt
-TdXXCOQMKAp+gCj/VByhmIr+0iNDC540gtvV303WpcbwnkkLYC0Ft2cYUyHtkstO
-fRcRO+K2cZozoSwVPyB8/J9RpcRK3jgnX9lujfwA/pAbP0J2UPQFxmWFRQnFjaq6
-rkqbNEBgLy+kFL1NEsRbvFbKrRi5bYy2lNms2NJPZvdNQbT/2dBZKmJqxHkxCuOQ
-FjhJQNeO+Njm1Z1iATS/3rts2yZlqXKsxQUzN6vNbD8KnXRMEeOXUYvbV4lqfCf8
-mS14WEbSiMy87GB5S9ucSV1XUrlTG5UGcMSZOBcEUpisRPEmQWUOTWIoDQ5FOia/
-GI+Ki523r2ruEmbmG37EBSBXdxIdndqrjy+QVAmCebyDx9eVEGOIpn26bW5LKeru
-mJxa/CFBaKi4bRvmdJRLAgMBAAGjQjBAMA4GA1UdDwEB/wQEAwIBBjAPBgNVHRMB
-Af8EBTADAQH/MB0GA1UdDgQWBBS182Xy/rAKkh/7PH3zRKCsYyXDFDANBgkqhkiG
-9w0BAQsFAAOCAgEAncDZNytDbrrVe68UT6py1lfF2h6Tm2p8ro42i87WWyP2LK8Y
-nLHC0hvNfWeWmjZQYBQfGC5c7aQRezak+tHLdmrNKHkn5kn+9E9LCjCaEsyIIn2j
-qdHlAkepu/C3KnNtVx5tW07e5bvIjJScwkCDbP3akWQixPpRFAsnP+ULx7k0aO1x
-qAeaAhQ2rgo1F58hcflgqKTXnpPM02intVfiVVkX5GXpJjK5EoQtLceyGOrkxlM/
-sTPq4UrnypmsqSagWV3HcUlYtDinc+nukFk6eR4XkzXBbwKajl0YjztfrCIHOn5Q
-CJL6TERVDbM/aAPly8kJ1sWGLuvvWYzMYgLzDul//rUF10gEMWaXVZV51KpS9DY/
-5CunuvCXmEQJHo7kGcViT7sETn6Jz9KOhvYcXkJ7po6d93A/jy4GKPIPnsKKNEmR
-xUuXY4xRdh45tMJnLTUDdC9FIU0flTeO9/vNpVA8OPU1i14vCz+MU8KX1bV3GXm/
-fxlB7VBBjX9v5oUep0o/j68R/iDlCOM4VVfRa8gX6T2FU7fNdatvGro7uQzIvWof
-gN9WUwCbEMBy/YhBSrXycKA8crgGg3x1mIsopn88JKwmMBa68oS7EHM9w7C4y71M
-7DiA+/9Qdp9RBWJpTS9i/mDnJg1xvo8Xz49mrrgfmcAXTCJqXi24NatI3Oc=
------END CERTIFICATE-----
------BEGIN CERTIFICATE-----
-MIICTjCCAdSgAwIBAgIRAIPgc3k5LlLVLtUUvs4K/QcwCgYIKoZIzj0EAwMwaDEL
-MAkGA1UEBhMCVVMxMzAxBgNVBAoTKihTVEFHSU5HKSBJbnRlcm5ldCBTZWN1cml0
-eSBSZXNlYXJjaCBHcm91cDEkMCIGA1UEAxMbKFNUQUdJTkcpIEJvZ3VzIEJyb2Nj
-b2xpIFgyMB4XDTIwMDkwNDAwMDAwMFoXDTQwMDkxNzE2MDAwMFowaDELMAkGA1UE
-BhMCVVMxMzAxBgNVBAoTKihTVEFHSU5HKSBJbnRlcm5ldCBTZWN1cml0eSBSZXNl
-YXJjaCBHcm91cDEkMCIGA1UEAxMbKFNUQUdJTkcpIEJvZ3VzIEJyb2Njb2xpIFgy
-MHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEOvS+w1kCzAxYOJbA06Aw0HFP2tLBLKPo
-FQqR9AMskl1nC2975eQqycR+ACvYelA8rfwFXObMHYXJ23XLB+dAjPJVOJ2OcsjT
-VqO4dcDWu+rQ2VILdnJRYypnV1MMThVxo0IwQDAOBgNVHQ8BAf8EBAMCAQYwDwYD
-VR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQU3tGjWWQOwZo2o0busBB2766XlWYwCgYI
-KoZIzj0EAwMDaAAwZQIwRcp4ZKBsq9XkUuN8wfX+GEbY1N5nmCRc8e80kUkuAefo
-uc2j3cICeXo1cOybQ1iWAjEA3Ooawl8eQyR4wrjCofUE8h44p0j7Yl/kBlJZT8+9
-vbtH7QiVzeKCOTQPINyRql6P
------END CERTIFICATE-----
------BEGIN CERTIFICATE-----
-MIIFWzCCA0OgAwIBAgIQTfQrldHumzpMLrM7jRBd1jANBgkqhkiG9w0BAQsFADBm
-MQswCQYDVQQGEwJVUzEzMDEGA1UEChMqKFNUQUdJTkcpIEludGVybmV0IFNlY3Vy
-aXR5IFJlc2VhcmNoIEdyb3VwMSIwIAYDVQQDExkoU1RBR0lORykgUHJldGVuZCBQ
-ZWFyIFgxMB4XDTIwMDkwNDAwMDAwMFoXDTI1MDkxNTE2MDAwMFowWTELMAkGA1UE
-BhMCVVMxIDAeBgNVBAoTFyhTVEFHSU5HKSBMZXQncyBFbmNyeXB0MSgwJgYDVQQD
-Ex8oU1RBR0lORykgQXJ0aWZpY2lhbCBBcHJpY290IFIzMIIBIjANBgkqhkiG9w0B
-AQEFAAOCAQ8AMIIBCgKCAQEAu6TR8+74b46mOE1FUwBrvxzEYLck3iasmKrcQkb+
-gy/z9Jy7QNIAl0B9pVKp4YU76JwxF5DOZZhi7vK7SbCkK6FbHlyU5BiDYIxbbfvO
-L/jVGqdsSjNaJQTg3C3XrJja/HA4WCFEMVoT2wDZm8ABC1N+IQe7Q6FEqc8NwmTS
-nmmRQm4TQvr06DP+zgFK/MNubxWWDSbSKKTH5im5j2fZfg+j/tM1bGaczFWw8/lS
-nukyn5J2L+NJYnclzkXoh9nMFnyPmVbfyDPOc4Y25aTzVoeBKXa/cZ5MM+WddjdL
-biWvm19f1sYn1aRaAIrkppv7kkn83vcth8XCG39qC2ZvaQIDAQABo4IBEDCCAQww
-DgYDVR0PAQH/BAQDAgGGMB0GA1UdJQQWMBQGCCsGAQUFBwMCBggrBgEFBQcDATAS
-BgNVHRMBAf8ECDAGAQH/AgEAMB0GA1UdDgQWBBTecnpI3zHDplDfn4Uj31c3S10u
-ZTAfBgNVHSMEGDAWgBS182Xy/rAKkh/7PH3zRKCsYyXDFDA2BggrBgEFBQcBAQQq
-MCgwJgYIKwYBBQUHMAKGGmh0dHA6Ly9zdGcteDEuaS5sZW5jci5vcmcvMCsGA1Ud
-HwQkMCIwIKAeoByGGmh0dHA6Ly9zdGcteDEuYy5sZW5jci5vcmcvMCIGA1UdIAQb
-MBkwCAYGZ4EMAQIBMA0GCysGAQQBgt8TAQEBMA0GCSqGSIb3DQEBCwUAA4ICAQCN
-DLam9yN0EFxxn/3p+ruWO6n/9goCAM5PT6cC6fkjMs4uas6UGXJjr5j7PoTQf3C1
-vuxiIGRJC6qxV7yc6U0X+w0Mj85sHI5DnQVWN5+D1er7mp13JJA0xbAbHa3Rlczn
-y2Q82XKui8WHuWra0gb2KLpfboYj1Ghgkhr3gau83pC/WQ8HfkwcvSwhIYqTqxoZ
-Uq8HIf3M82qS9aKOZE0CEmSyR1zZqQxJUT7emOUapkUN9poJ9zGc+FgRZvdro0XB
-yphWXDaqMYph0DxW/10ig5j4xmmNDjCRmqIKsKoWA52wBTKKXK1na2ty/lW5dhtA
-xkz5rVZFd4sgS4J0O+zm6d5GRkWsNJ4knotGXl8vtS3X40KXeb3A5+/3p0qaD215
-Xq8oSNORfB2oI1kQuyEAJ5xvPTdfwRlyRG3lFYodrRg6poUBD/8fNTXMtzydpRgy
-zUQZh/18F6B/iW6cbiRN9r2Hkh05Om+q0/6w0DdZe+8YrNpfhSObr/1eVZbKGMIY
-qKmyZbBNu5ysENIK5MPc14mUeKmFjpN840VR5zunoU52lqpLDua/qIM8idk86xGW
-xx2ml43DO/Ya/tVZVok0mO0TUjzJIfPqyvr455IsIut4RlCR9Iq0EDTve2/ZwCuG
-hSjpTUFGSiQrR2JK2Evp+o6AETUkBCO1aw0PpQBPDQ==
------END CERTIFICATE-----
------BEGIN CERTIFICATE-----
-MIIDCzCCApGgAwIBAgIRALRY4992FVxZJKOJ3bpffWIwCgYIKoZIzj0EAwMwaDEL
-MAkGA1UEBhMCVVMxMzAxBgNVBAoTKihTVEFHSU5HKSBJbnRlcm5ldCBTZWN1cml0
-eSBSZXNlYXJjaCBHcm91cDEkMCIGA1UEAxMbKFNUQUdJTkcpIEJvZ3VzIEJyb2Nj
-b2xpIFgyMB4XDTIwMDkwNDAwMDAwMFoXDTI1MDkxNTE2MDAwMFowVTELMAkGA1UE
-BhMCVVMxIDAeBgNVBAoTFyhTVEFHSU5HKSBMZXQncyBFbmNyeXB0MSQwIgYDVQQD
-ExsoU1RBR0lORykgRXJzYXR6IEVkYW1hbWUgRTEwdjAQBgcqhkjOPQIBBgUrgQQA
-IgNiAAT9v/PJUtHOTk28nXCXrpP665vI4Z094h8o7R+5E6yNajZa0UubqjpZFoGq
-u785/vGXj6mdfIzc9boITGusZCSWeMj5ySMZGZkS+VSvf8VQqj+3YdEu4PLZEjBA
-ivRFpEejggEQMIIBDDAOBgNVHQ8BAf8EBAMCAYYwHQYDVR0lBBYwFAYIKwYBBQUH
-AwIGCCsGAQUFBwMBMBIGA1UdEwEB/wQIMAYBAf8CAQAwHQYDVR0OBBYEFOv5JcKA
-KGbibQiSMvPC4a3D/zVFMB8GA1UdIwQYMBaAFN7Ro1lkDsGaNqNG7rAQdu+ul5Vm
-MDYGCCsGAQUFBwEBBCowKDAmBggrBgEFBQcwAoYaaHR0cDovL3N0Zy14Mi5pLmxl
-bmNyLm9yZy8wKwYDVR0fBCQwIjAgoB6gHIYaaHR0cDovL3N0Zy14Mi5jLmxlbmNy
-Lm9yZy8wIgYDVR0gBBswGTAIBgZngQwBAgEwDQYLKwYBBAGC3xMBAQEwCgYIKoZI
-zj0EAwMDaAAwZQIwXcZbdgxcGH9rTErfSTkXfBKKygU0yO7OpbuNeY1id0FZ/hRY
-N5fdLOGuc+aHfCsMAjEA0P/xwKr6NQ9MN7vrfGAzO397PApdqfM7VdFK18aEu1xm
-3HMFKzIR8eEPsMx4smMl
------END CERTIFICATE-----

src/haproxy-sidecar/Dockerfile

@@ -0,0 +1,7 @@
+FROM docker
+
+COPY balena.sh /usr/local/bin/balena.sh
+
+RUN apk add --no-cache bash
+
+CMD /usr/local/bin/balena.sh

src/haproxy-sidecar/balena.sh

@@ -0,0 +1,75 @@
+#!/usr/bin/env bash
+
+set -ea
+
+[[ $VERBOSE =~ on|On|Yes|yes|true|True ]] && set -x
+
+function cleanup() {
+   rm -f /host/run/docker.sock
+
+   # crash loop backoff
+   sleep "$(( (RANDOM % 5) + 5 ))s"
+}
+
+trap 'cleanup' EXIT
+
+if [ -S /host/run/balena-engine.sock ]; then
+    ln -s /host/run/balena-engine.sock /host/run/docker.sock
+fi
+
+which curl || apk add curl --no-cache
+which jq || apk add jq --no-cache
+
+if docker inspect "${BALENA_APP_UUID}_default" --format "{{.ID}}"; then
+    network="${BALENA_APP_UUID}_default"
+elif docker inspect "${BALENA_APP_ID}_default" --format "{{.ID}}"; then
+    network="${BALENA_APP_ID}_default"
+else
+    network=open-balena_default
+fi
+
+# shellcheck disable=SC2153
+for alias in ${ALIASES//,/ }; do
+    hostname="${alias}.${DNS_TLD}"
+    aliases="--alias ${hostname} ${aliases}"
+done
+
+while true; do
+    if [[ -n $BALENA_SUPERVISOR_ADDRESS ]] && [[ -n $BALENA_SUPERVISOR_API_KEY ]]; then
+        while [[ "$(curl --silent --retry 3 --fail \
+          "${BALENA_SUPERVISOR_ADDRESS}/v1/device?apikey=${BALENA_SUPERVISOR_API_KEY}" \
+          -H "Content-Type:application/json" | jq -r '.update_pending')" =~ true ]]; do
+            sleep "$(( (RANDOM % 3) + 3 ))s"
+        done
+        sleep "$(( (RANDOM % 5) + 5 ))s"
+    fi
+
+    while [[ "$(docker ps \
+      --filter "name=haproxy" \
+      --filter "expose=1936/tcp" \
+      --filter "status=running" \
+      --filter "network=${network}" \
+      --format "{{.ID}}")" == '' ]]; do
+        sleep "$(( (RANDOM % 3) + 3 ))s"
+    done
+
+    haproxy="$(docker ps \
+      --filter "name=haproxy" \
+      --filter "expose=1936/tcp" \
+      --filter "status=running" \
+      --filter "network=${network}" \
+      --format "{{.ID}}")"
+
+    if ! [[ $restarted == "${haproxy}" ]]; then
+        docker network disconnect "${network}" "${haproxy}"
+
+        # shellcheck disable=SC2086
+        docker network connect --alias haproxy ${aliases} "${network}" "${haproxy}"
+
+        docker restart "${haproxy}"
+
+        restarted="${haproxy}"
+    fi
+
+    sleep "$(( (RANDOM % 15) + 15 ))s"
+done

src/haproxy/Dockerfile

@@ -1,10 +1,4 @@
-FROM haproxy:1.9-alpine
+# https://github.com/balena-io/open-balena-haproxy
-
+FROM balena/open-balena-haproxy:v4.3.2
-VOLUME [ "/certs" ]
-
-RUN apk add --update inotify-tools
 
 COPY haproxy.cfg /usr/local/etc/haproxy/haproxy.cfg
-COPY start-haproxy.sh /start-haproxy
-
-CMD /start-haproxy

src/haproxy/haproxy.cfg

@@ -1,134 +1,176 @@
 global
-  tune.ssl.default-dh-param 1024
+	tune.ssl.default-dh-param 1024
+	# https://github.com/haproxytech/haproxy-lua-cors
+	lua-load /usr/local/etc/haproxy/cors.lua
+	# https://www.haproxy.com/blog/introduction-to-haproxy-logging/
+	log stdout format raw daemon "${LOGLEVEL}"
+	log stderr format raw daemon "${LOGLEVEL}"
+	ssl-default-bind-options ssl-min-ver TLSv1.2
 
 defaults
-  timeout connect 5s
+	balance roundrobin
-  timeout client 50s
+	default-server init-addr last,libc,none
-  timeout server 50s
+	default-server inter 3s rise 2 fall 3
+	log global
+	mode http
+	option contstats
+	option dontlognull
+	option forwardfor
+	option httplog
+	timeout client 63s
+	timeout connect 5s
+	timeout http-keep-alive 1s
+	timeout http-request 63s
+	timeout server 63s
+	timeout tunnel 3600s
 
-frontend http-in
+resolvers docker-bridge-resolver
-  mode http
+	nameserver docker-resolver 127.0.0.11:53
-  option forwardfor
+	hold valid 0ms
-  bind *:80
-  reqadd X-Forwarded-Proto:\ http
 
-  acl is_cert_validation path -i -m beg "/.well-known/acme-challenge/"
+http-errors balena-http-errors
-  use_backend cert-provider if is_cert_validation
+	errorfile 400 /etc/haproxy/errors/400.http
+	errorfile 401 /etc/haproxy/errors/401.http
+	errorfile 403 /etc/haproxy/errors/403.http
+	errorfile 404 /etc/haproxy/errors/404.http
+	errorfile 500 /etc/haproxy/errors/500.http
+	errorfile 502 /etc/haproxy/errors/502.http
+	errorfile 503 /etc/haproxy/errors/503.http
 
-  acl host_api hdr_dom(host) -i "api.${HAPROXY_HOSTNAME}"
+userlist balena
-  use_backend backend_api if host_api
+	user balena insecure-password "${BALENA_DEVICE_UUID}"
 
-  acl host_registry hdr_dom(host) -i "registry.${HAPROXY_HOSTNAME}"
+listen haproxy-stats
-  use_backend backend_registry if host_registry
+	bind :::1936 v4v6 ssl crt "${CERT_CHAIN_PATH}" alpn h2,http/1.1
+	stats auth "balena:${BALENA_DEVICE_UUID}"
+	stats enable
+	stats uri /metrics
 
-  acl host_vpn hdr_dom(host) -i "vpn.${HAPROXY_HOSTNAME}"
+frontend http
-  use_backend backend_vpn if host_vpn
+	bind :::80 v4v6
+	default_backend api-backend
+	errorfiles balena-http-errors
+	http-request capture req.hdr(Host) len 15
+	http-response lua.cors
+	# https://www.haproxy.com/blog/haproxy-log-customization/
+	log-format "%ci:%cp [%tr] %ft %b/%s %TR/%Tw/%Tc/%Tr/%Ta %ST %B %CC %CS %tsc %ac/%fc/%bc/%sc/%rc %sq/%bq %hr %hs %{+Q}r"
 
-  acl host_s3 hdr_dom(host) -i "s3.${HAPROXY_HOSTNAME}"
+	acl api_dead nbsrv(api-backend) lt 1
-  use_backend backend_s3 if host_s3
+	acl registry_dead nbsrv(registry-backend) lt 1
+	acl vpn_dead nbsrv(vpn-backend) lt 1
+	monitor-uri /health
+	monitor fail if api_dead registry_dead vpn_dead
 
-frontend ssl-in
+	acl host-api-backend hdr_beg(host) -i "api."
-  mode tcp
+	# default public device URL(s) always go to the API
-  bind *:443
+	acl host-pdu-default hdr(host) -m reg -i "\.?([0-9a-f]{32}|${BALENA_DEVICE_UUID})\.(devices|balena-?(.*)-devices)\."
-  tcp-request inspect-delay 2s
+	use_backend api-backend if host-api-backend || host-pdu-default
-  tcp-request content accept if { req.ssl_hello_type 1 }
 
-  acl is_ssl req.ssl_ver 2:3.4
+	acl host-registry-backend hdr_beg(host) -i "registry2."
+	http-request add-header X-Forwarded-Proto http if host-registry-backend
+	use_backend registry-backend if host-registry-backend
 
-  acl host_tunnel req_ssl_sni -i "tunnel.${HAPROXY_HOSTNAME}"
+	acl host-s3-backend hdr_beg(host) -i "s3."
-  use_backend redirect-to-tunnel-in if host_tunnel
+	http-request add-header X-Forwarded-Proto http if host-s3-backend
+	use_backend s3-backend if host-s3-backend
 
-  use_backend redirect-to-https-in if is_ssl
+	acl host-minio-backend hdr_beg(host) -i "minio."
-  use_backend vpn-devices if !is_ssl
+	http-request add-header X-Forwarded-Proto http if host-minio-backend
+	use_backend minio-backend if host-minio-backend
 
-backend redirect-to-https-in
+# routes between OpenVPN, SSL and HTTPS traffic
-  mode tcp
+frontend tcp-router
-  balance roundrobin
+	mode tcp
-  server localhost 127.0.0.1:444 send-proxy-v2
+	option tcplog
+	log-format "%ci:%cp [%t] %ft %b/%s %Tw/%Tc/%Tt %B %ts %ac/%fc/%bc/%sc/%rc %sq/%bq"
+	bind :::443 v4v6
+	tcp-request inspect-delay 2s
+	tcp-request content accept if { req.ssl_hello_type 1 }
+	acl is_ssl req.ssl_ver 2:3.4
 
-backend redirect-to-tunnel-in
+	acl sni-host-tunnel req_ssl_sni -m beg "tunnel."
-  mode tcp
+	use_backend redirect-to-tunnel if sni-host-tunnel
-  balance roundrobin
-  server localhost 127.0.0.1:3129
 
-frontend https-in
+	# everything else => HTTPS
-  mode http
+	use_backend redirect-to-https if is_ssl
-  option forwardfor
-  bind 127.0.0.1:444 ssl crt /etc/ssl/private/open-balena.pem accept-proxy
-  reqadd X-Forwarded-Proto:\ https
 
-  acl host_api hdr_dom(host) -i "api.${HAPROXY_HOSTNAME}"
+	# or VPN
-  use_backend backend_api if host_api
+	use_backend vpn-backend if !is_ssl
 
-  acl host_registry hdr_dom(host) -i "registry.${HAPROXY_HOSTNAME}"
+backend redirect-to-tunnel
-  use_backend backend_registry if host_registry
+	mode tcp
+	server localhost 127.0.0.1:3129 send-proxy-v2
 
-  acl host_vpn hdr_dom(host) -i "vpn.${HAPROXY_HOSTNAME}"
+# https://stackoverflow.com/a/39213442/1559300
-  use_backend backend_vpn if host_vpn
+listen tunnel-backend
+	mode tcp
+	option tcplog
+	log-format "%ci:%cp [%t] %ft %b/%s %Tw/%Tc/%Tt %B %ts %ac/%fc/%bc/%sc/%rc %sq/%bq"
+	bind 127.0.0.1:3129 ssl crt "${CERT_CHAIN_PATH}" alpn h2,http/1.1 accept-proxy
+	server tunnel vpn:3128 resolvers docker-bridge-resolver resolve-prefer ipv4 check port 3128
 
-  acl host_s3 hdr_dom(host) -i "s3.${HAPROXY_HOSTNAME}"
+backend redirect-to-https
-  use_backend backend_s3 if host_s3
+	mode tcp
+	server localhost 127.0.0.1:444 send-proxy-v2
 
-backend backend_api
+frontend https
-  mode http
+	bind 127.0.0.1:444 ssl crt "${CERT_CHAIN_PATH}" alpn h2,http/1.1 accept-proxy
-  option forwardfor
+	default_backend api-backend
-  balance roundrobin
+	errorfiles balena-http-errors
-  server balena_api_1 api:80 check port 80
+	http-request add-header X-Forwarded-Proto https
+	http-request capture req.hdr(Host) len 15
+	http-response lua.cors
+	log-format "%ci:%cp [%tr] %ft %b/%s %TR/%Tw/%Tc/%Tr/%Ta %ST %B %CC %CS %tsc %ac/%fc/%bc/%sc/%rc %sq/%bq %hr %hs %{+Q}r"
 
-backend backend_registry
+	acl host-api-backend hdr_beg(host) -i "api."
-  mode http
+	use_backend api-backend if host-api-backend
-  option forwardfor
-  balance roundrobin
-  server balena_registry_1 registry:80 check port 80
 
-backend backend_vpn
+	acl host-registry-backend hdr_beg(host) -i "registry2."
-  mode http
+	use_backend registry-backend if host-registry-backend
-  option forwardfor
-  balance roundrobin
-  server balena_vpn_1 vpn:80 check port 80
 
-backend backend_s3
+	acl host-s3-backend hdr_beg(host) -i "s3."
-  mode http
+	use_backend s3-backend if host-s3-backend
-  option forwardfor
-  balance roundrobin
-  server balena_s3_1 s3:80 check port 80
 
-backend cert-provider
+	acl host-minio-backend hdr_beg(host) -i "minio."
-  mode http
+	use_backend minio-backend if host-minio-backend
-  option forwardfor
-  balance roundrobin
-  server balena_cert-provider_1 cert-provider:80 no-check
 
-backend vpn-devices
+	acl host-ca-backend hdr_beg(host) -i "ca."
-  mode tcp
+	# only allow CRL requests unauthenticated, protect everything else
-  server balena_vpn_1 vpn:443 send-proxy-v2 check-send-proxy port 443
+	acl balena-ca-crl path -i -m beg /api/v1/cfssl/crl
+	acl balena-ca-auth http_auth(balena)
+	http-request auth realm balena-ca if host-ca-backend !balena-ca-auth !balena-ca-crl
+	use_backend ca-backend if host-ca-backend
 
-frontend db
+	acl host-ocsp-backend hdr_beg(host) -i "ocsp."
-  mode tcp
+	use_backend ocsp-backend if host-ocsp-backend
-  bind *:5432
-  default_backend backend_db
-  timeout client 1h
 
-backend backend_db
+backend api-backend
-  mode tcp
+	server api api:80 resolvers docker-bridge-resolver resolve-prefer ipv4 check port 80
-  server balena_db_1 db:5432 check port 5432
 
-frontend redis
+backend registry-backend
-  mode tcp
+	server registry registry:80 resolvers docker-bridge-resolver resolve-prefer ipv4 check port 80
-  bind *:6379
-  default_backend backend_redis
-  timeout client 1h
 
-backend backend_redis
+backend s3-backend
-  mode tcp
+	server s3 s3:80 resolvers docker-bridge-resolver resolve-prefer ipv4 check port 80
-  server balena_redis_1 redis:6379 check port 6379
 
-listen vpn-tunnel
+# https://github.com/minio/console
-  mode tcp
+backend minio-backend
-  bind *:3128
+	server s3-console s3:43697 resolvers docker-bridge-resolver resolve-prefer ipv4 check port 43697
-  server balena_vpn vpn:3128 check port 3128
 
-listen vpn-tunnel-tls
+backend db-backend
-  mode tcp
+	mode tcp
-  bind *:3129 ssl crt /etc/ssl/private/open-balena.pem
+	server db db:5432 resolvers docker-bridge-resolver resolve-prefer ipv4 check port 5432
-  server balena_vpn vpn:3128 check port 3128
+
+backend redis-backend
+	mode tcp
+	server redis redis:6379 resolvers docker-bridge-resolver resolve-prefer ipv4 check port 6379
+
+backend ca-backend
+	server cfssl-ca balena-ca:8888 resolvers docker-bridge-resolver resolve-prefer ipv4 check port 8888
+
+backend ocsp-backend
+	server cfssl-ocsp balena-ca:8889 resolvers docker-bridge-resolver resolve-prefer ipv4 check port 8889
+
+backend vpn-backend
+	mode tcp
+	server openvpn vpn:443 resolvers docker-bridge-resolver resolve-prefer ipv4 send-proxy-v2 check-send-proxy check port 443

src/haproxy/start-haproxy.sh

@@ -1,32 +0,0 @@
-#!/bin/sh
-
-OPENBALENA_CERT=/etc/ssl/private/open-balena.pem
-mkdir -p "$(dirname "${OPENBALENA_CERT}")"
-
-if [ -f "/certs/open-balena.pem" ]; then
-    echo "Using certificate from cert-provider..."
-    cp /certs/open-balena.pem "${OPENBALENA_CERT}"
-else
-    echo "Building certificate from environment variables..."
-    (
-        echo "${BALENA_HAPROXY_CRT}" | base64 -d
-        echo "${BALENA_HAPROXY_KEY}" | base64 -d
-        echo "${BALENA_ROOT_CA}" | base64 -d
-    ) > "${OPENBALENA_CERT}"
-fi
-
-haproxy -f /usr/local/etc/haproxy/haproxy.cfg -W &
-HAPROXY_PID=$!
-
-while true; do
-    inotifywait -r -e create -e modify -e delete /certs
-    
-    if [ -f "/certs/open-balena.pem" ]; then
-        echo "Updating certificate from cert-provider..."
-        cp /certs/open-balena.pem "${OPENBALENA_CERT}"
-    fi
-    
-    echo "Certificate change detected. Reloading..."
-    kill -SIGUSR2 $HAPROXY_PID
-    sleep 1;
-done

src/tag-sidecar/Dockerfile

@@ -0,0 +1,5 @@
+FROM bash:alpine3.19
+
+COPY balena.sh /usr/local/bin/balena.sh
+
+CMD /usr/local/bin/balena.sh

src/tag-sidecar/balena.sh

@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+
+set -ea
+
+[[ $VERBOSE =~ on|On|Yes|yes|true|True ]] && set -x
+
+[[ $ENABLED == 'true' ]] || exit
+
+curl_with_opts() {
+    curl --fail --silent --retry 3 --connect-timeout 3 --compressed "$@"
+}
+
+get_aws_meta() {
+    if [[ $1 =~ ^.*/$ ]]; then
+        for key in $(curl_with_opts "$1"); do
+            get_aws_meta "$1${key}"
+        done
+    else
+        echo "$(echo "$1" | cut -c41-);$(curl_with_opts "$1" | tr '\n' ',')"
+    fi
+}
+
+if [[ -n $BALENA_API_URL ]] && [[ -n $BALENA_DEVICE_UUID ]] && [[ -n $BALENA_API_KEY ]]; then
+	which curl || apk add curl --no-cache
+	which jq || apk add jq --no-cache
+
+	device_id="$(curl_with_opts \
+	  "${BALENA_API_URL}/v6/device?\$filter=uuid%20eq%20'${BALENA_DEVICE_UUID}'" \
+	  -H "Content-Type: application/json" \
+	  -H "Authorization: Bearer ${BALENA_API_KEY}" | jq -r .d[].id)"
+
+	for key in $(curl_with_opts http://169.254.169.254/latest/meta-data \
+	  | grep -Ev 'iam|metrics|identity-credentials|network|events'); do
+		for kv in $(get_aws_meta "http://169.254.169.254/latest/meta-data/${key}"); do
+			tag_key="$(echo "${kv}" | awk -F';' '{print $1}')"
+			value="$(echo "${kv}" | awk -F';' '{print $2}')"
+
+			curl_with_opts "${BALENA_API_URL}/v6/device_tag" \
+			  -H "Content-Type: application/json" \
+			  -H "Authorization: Bearer ${BALENA_API_KEY}" \
+			  --data "{\"device\":\"${device_id}\",\"tag_key\":\"${tag_key}\",\"value\":\"${value}\"}"
+		done
+	done
+fi

src/test-device/Dockerfile

@@ -0,0 +1,11 @@
+# https://hub.docker.com/r/qemux/qemu-docker
+# https://github.com/qemus/qemu-docker
+FROM qemux/qemu-docker:5.16
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+	minicom \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY balena.sh /usr/sbin/
+
+WORKDIR /balena

src/test-device/balena.sh

@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+
+set -ae
+
+[[ $VERBOSE =~ on|On|Yes|yes|true|True ]] && set -x
+
+function cleanup() {
+   rm -f "${tmpimg}"
+
+   # crash loop backoff
+   sleep "$(( (RANDOM % 5) + 5 ))s"
+}
+trap 'cleanup' EXIT
+
+if [[ "$PRODUCTION_MODE" =~ true ]]; then
+    exit
+fi
+
+until test -f "${GUEST_IMAGE%.*}.ready"; do sleep "$(( (RANDOM % 5) + 5 ))s"; done
+
+tmpimg="$(mktemp)"
+cat <"${GUEST_IMAGE}" >"${tmpimg}"
+
+exec /usr/bin/qemu-system-x86_64 \
+  -bios /usr/share/ovmf/OVMF.fd \
+  -chardev socket,id=serial0,path=/run/console.sock,server=on,wait=off \
+  -cpu max \
+  -device ahci,id=ahci \
+  -device ide-hd,drive=disk,bus=ahci.0 \
+  -device virtio-net-pci,netdev=n1 \
+  -drive file="${tmpimg}",media=disk,cache=none,format=raw,if=none,id=disk \
+  -m "${MEMORY}" \
+  -machine q35 \
+  -netdev "user,id=n1,dns=127.0.0.1,guestfwd=tcp:10.0.2.100:80-cmd:netcat haproxy 80,guestfwd=tcp:10.0.2.100:443-cmd:netcat haproxy 443" \
+  -nodefaults \
+  -nographic \
+  -serial chardev:serial0 \
+  -smp "${CPU}"