v4.0.3

Update aws-actions/configure-aws-credentials digest to 43c8912

.github/CODEOWNERS

@@ -1,2 +0,0 @@
-# Main repo owners:
-*       @dfunckt @richbayliss

.github/workflows/flowzone.yml

@@ -0,0 +1,57 @@
+name: Flowzone
+
+on:
+  pull_request:
+    types: [opened, synchronize, closed]
+    branches: [main, master]
+  pull_request_target:
+    types: [opened, synchronize, closed]
+    branches: [main, master]
+
+jobs:
+  flowzone:
+    name: Flowzone
+    uses: product-os/flowzone/.github/workflows/flowzone.yml@master
+    # prevent duplicate workflow executions for pull_request and pull_request_target
+    if: |
+      (
+        github.event.pull_request.head.repo.full_name == github.repository &&
+        github.event_name == 'pull_request'
+      ) || (
+        github.event.pull_request.head.repo.full_name != github.repository &&
+        github.event_name == 'pull_request_target'
+      )
+    secrets: inherit
+    with:
+      jobs_timeout_minutes: 60
+      cloudflare_website: open-balena
+      custom_runs_on: |
+        [
+          [
+            "self-hosted",
+            "Linux",
+            "X64"
+          ]
+        ]
+
+      balena_slugs: |
+        balena/open-balena
+
+  tests:
+    name: openBalena tests
+    uses: ./.github/workflows/tests.yml
+    needs: [flowzone]
+    if: |
+      ((
+        github.event.pull_request.head.repo.full_name == github.repository &&
+        github.event_name == 'pull_request'
+      ) || (
+        github.event.pull_request.head.repo.full_name != github.repository &&
+        github.event_name == 'pull_request_target'
+      )) && github.event.action != 'closed'
+    secrets: inherit
+    with:
+      environment: balena-cloud.com
+      fleet: balena/open-balena
+      # https://dash.cloudflare.com/001b3ed2352612aaa068aca1b0022736/balena-devices.com/dns
+      dns_tld: balena-devices.com

.github/workflows/tests.yml

@@ -0,0 +1,676 @@
+---
+name: openBalena tests
+
+on:
+  workflow_call:
+    inputs:
+      environment:
+        description: "balenaCloud environment"
+        required: true
+        type: string
+      fleet:
+        description: "balenaCloud fleet"
+        required: true
+        type: string
+      dns_tld:
+        description: "domain name to use for issuing SSL certificates"
+        required: true
+        type: string
+
+# https://docs.github.com/en/actions/security-guides/automatic-token-authentication
+# https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#permissions
+permissions:
+  actions: read
+  checks: read
+  contents: read
+  deployments: read
+  id-token: write # AWS GitHub OIDC required: write
+  issues: read
+  discussions: read
+  packages: read
+  pages: read
+  pull-requests: read
+  repository-projects: read
+  security-events: read
+  statuses: read
+
+env:
+  # Stack ID
+  # arn:aws:cloudformation:us-east-1:491725000532:stack/balena-tests-s3-certs/814dea60-404d-11ed-b06f-0a7d458f8ba5
+  AWS_S3_CERTS_BUCKET: balena-tests-certs
+  # (kvm) nested virtualisation not supported on AWS/EC2 instance types|classes other than X.metal
+  AWS_EC2_INSTANCE_TYPE: c6a.2xlarge
+  AWS_EC2_LAUNCH_TEMPLATE: lt-02e10a4f66261319d
+  AWS_EC2_LT_VERSION: 2
+  AWS_IAM_USERNAME: balena-tests-iam-User-1GXO3XP12N6LL
+  AWS_VPC_SECURITY_GROUP_IDS: sg-057937f4d89d9d51c
+  AWS_VPC_SUBNET_IDS: 'subnet-02d18a08ea4058574 subnet-0a026eae1df907a09'
+  # otherwise it tries to send data to an endpoint provided by a private project
+  # https://github.com/balena-io/analytics-backend
+  # .. which is not part of openBalena
+  BALENARC_NO_ANALYTICS: '1'  # https://github.com/balena-io/balena-cli/blob/master/lib/events.ts#L62-L70
+  DEBUG: '0'  # https://github.com/balena-io/balena-cli/issues/2447
+  RETRY: 3
+  SUBDOMAIN: auto
+
+jobs:
+  test:
+    runs-on: ["self-hosted", "X64", "distro:jammy"]  # tests require socat v1.7.4
+    timeout-minutes: 60
+    strategy:
+      fail-fast: true
+
+    steps:
+    - uses: actions/checkout@b80ff79f1755d06ba70441c368a6fe801f5f3a62
+      with:
+        # FIXME: remove once balenaBlocks/balenaVirt is a thing
+        submodules: true
+
+    - uses: aws-actions/configure-aws-credentials@43c891271eabca0b34d95f2453e2fe354928b16d
+      with:
+        aws-region: ${{ vars.AWS_REGION || 'us-east-1' }}
+        role-session-name: github-${{ github.job }}-${{ github.run_id }}-${{ github.run_attempt }}
+        # balena-io/environments-bases: aws/balenacloud/ephemeral-tests/balena-tests-iam.yml
+        role-to-assume: ${{ vars.AWS_IAM_ROLE }}
+
+    # https://github.com/pdcastro/ssh-uuid#why
+    # https://github.com/pdcastro/ssh-uuid#linux-debian-ubuntu-others
+    - name: install additional dependencies
+      shell: bash
+      run: |
+        set -ue
+        echo '::notice::install additional dependencies'
+
+        [[ '${{ vars.VERBOSE || 'false' }}' =~ on|On|Yes|yes|true|True ]] && set -x
+
+        source src/balena-tests/functions
+
+        mkdir -p "${RUNNER_TEMP}/ssh-uuid"
+
+        wget -q -O "${RUNNER_TEMP}/ssh-uuid/ssh-uuid" https://raw.githubusercontent.com/pdcastro/ssh-uuid/master/ssh-uuid.sh \
+          && chmod +x "${RUNNER_TEMP}/ssh-uuid/ssh-uuid" \
+          && ln -s "${RUNNER_TEMP}/ssh-uuid/ssh-uuid" "${RUNNER_TEMP}/ssh-uuid/scp-uuid"
+
+        with_backoff balena login --token '${{ secrets.BALENA_API_KEY }}'
+
+        balena version
+
+        "${RUNNER_TEMP}/ssh-uuid/scp-uuid" --help
+
+        grep -q "${RUNNER_TEMP}/ssh-uuid" "${GITHUB_PATH}" \
+          || echo "${RUNNER_TEMP}/ssh-uuid" >> "${GITHUB_PATH}"
+
+    - name: (pre)register test device
+      id: register-test-device
+      if: ${{ github.event_name == 'pull_request' && github.event.action != 'closed'}}
+      run: |
+        set -ue
+
+        [[ '${{ vars.VERBOSE || 'false' }}' =~ on|On|Yes|yes|true|True ]] && set -x
+
+        source src/balena-tests/functions
+
+        with_backoff balena login --token '${{ secrets.BALENA_API_KEY }}'
+
+        balena_device_uuid="$(openssl rand -hex 16)"
+
+        # https://www.balena.io/docs/learn/more/masterclasses/advanced-cli/#52-preregistering-a-device
+        balena device register '${{ inputs.fleet }}' --uuid "${balena_device_uuid}"
+
+        device_id="$(balena device "${balena_device_uuid}" | grep ^ID: | cut -c20-)"
+
+        # the actual version deployed depends on the AWS EC2/AMI, defined in AWS_EC2_LAUNCH_TEMPLATE
+        os_version="$(balena os versions ${{ vars.DEVICE_TYPE || 'generic-amd64' }} | head -n 1)"
+
+        balena config generate \
+          --version "${os_version}" \
+          --device "${balena_device_uuid}" \
+          --network ethernet \
+          --appUpdatePollInterval 10 \
+          $([[ '${{ vars.DEVELOPMENT_MODE || 'false' }}' =~ true ]] && echo '--dev') \
+          --output config.json
+
+        balena tag set balena ephemeral-test-device --device "${balena_device_uuid}"
+
+        github_vars=(GITHUB_ACTOR GITHUB_BASE_REF GITHUB_HEAD_REF GITHUB_JOB \
+          GITHUB_REF GITHUB_REF_NAME GITHUB_REF_TYPE GITHUB_REPOSITORY \
+          GITHUB_REPOSITORY_OWNER GITHUB_RUN_ATTEMPT GITHUB_RUN_ID GITHUB_RUN_NUMBER \
+          GITHUB_SHA GITHUB_WORKFLOW RUNNER_ARCH RUNNER_NAME RUNNER_OS)
+
+        for github_var in "${github_vars[@]}"; do
+            balena tag set ${github_var} "${!github_var}" --device "${balena_device_uuid}"
+        done
+
+        echo "balena_device_uuid=${balena_device_uuid}" >> "${GITHUB_OUTPUT}"
+        echo "balena_device_id=${device_id}" >> "${GITHUB_OUTPUT}"
+
+   # https://github.com/balena-io/balena-cli/issues/1543
+    - name: pin device to draft release
+      if: ${{ github.event_name == 'pull_request' && github.event.action != 'closed'}}
+      run: |
+        set -uae
+
+        [[ '${{ vars.VERBOSE || 'false' }}' =~ on|On|Yes|yes|true|True ]] && set -x
+
+        source src/balena-tests/functions
+
+        with_backoff balena login --token '${{ secrets.BALENA_API_KEY }}'
+
+        pr_id='${{ github.event.pull_request.id }}'
+        head_sha='${{ github.event.pull_request.head.sha || github.event.head_commit.id }}'
+        release_id="$(with_backoff balena releases '${{ inputs.fleet }}' --json \
+          | jq -r --arg pr_id "${pr_id}" --arg head_sha "${head_sha}" '.[]
+          | select(.release_tag[].tag_key=="balena-ci-commit-sha")
+          | select(.release_tag[].value==$head_sha)
+          | select(.release_tag[].tag_key=="balena-ci-id")
+          | select(.release_tag[].value==$pr_id).commit')"
+
+        with_backoff balena device pin \
+          ${{ steps.register-test-device.outputs.balena_device_uuid }} \
+          "${release_id}"
+
+        with_backoff balena device ${{ steps.register-test-device.outputs.balena_device_uuid }}
+
+    - name: configure test device environment
+      if: ${{ github.event_name == 'pull_request' && github.event.action != 'closed'}}
+      run: |
+        set -ue
+
+        [[ '${{ vars.VERBOSE || 'false' }}' =~ on|On|Yes|yes|true|True ]] && set -x
+
+        source src/balena-tests/functions
+
+        with_backoff balena login --token '${{ secrets.BALENA_API_KEY }}'
+
+        with_backoff balena env add VERBOSE "${{ vars.VERBOSE || 'false' }}" \
+          --device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
+
+        with_backoff balena env add BALENARC_NO_ANALYTICS '1' \
+          --device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
+
+        with_backoff balena env add DNS_TLD '${{ env.SUBDOMAIN }}.${{ inputs.dns_tld }}' \
+          --device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
+
+        with_backoff balena env add DB_HOST db \
+          --service api \
+          --device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
+
+        with_backoff balena env add REDIS_HOST redis:6379 \
+          --service api \
+          --device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
+
+
+        # to allow devices running locally to communicate to the local API, we can route
+        # to the local Docker network aliases instead of public DNS, since (a) DNS_TLD is
+        # guestfwd(ed) in QEMU to a special internal IP 10.0.2.100; (b) is proxied to
+        # haproxy network alias on device; and (c) made public with a wildcard DNS record
+        # (e.g.)
+        #
+        # $ dig +short api.auto.balena-devices.com
+        # 10.0.2.100
+        #
+
+        with_backoff balena env add API_HOST 'api.${{ env.SUBDOMAIN }}.${{ inputs.dns_tld }}' \
+          --device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
+
+        # not used but required for config.json to be valid
+        with_backoff balena env add DELTA_HOST 'delta.${{ env.SUBDOMAIN }}.${{ inputs.dns_tld }}' \
+          --device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
+
+        with_backoff balena env add REGISTRY2_HOST 'registry2.${{ env.SUBDOMAIN }}.${{ inputs.dns_tld }}' \
+          --device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
+
+        with_backoff balena env add VPN_HOST 'cloudlink.${{ env.SUBDOMAIN }}.${{ inputs.dns_tld }}' \
+          --device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
+
+        with_backoff balena env add HOST 'api.${{ env.SUBDOMAIN }}.${{ inputs.dns_tld }}' \
+          --service api \
+          --device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
+
+        with_backoff balena env add TOKEN_AUTH_CERT_ISSUER 'api.${{ env.SUBDOMAIN }}.${{ inputs.dns_tld }}' \
+          --service api \
+          --device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
+
+        with_backoff balena env add REGISTRY2_TOKEN_AUTH_ISSUER 'api.${{ env.SUBDOMAIN }}.${{ inputs.dns_tld }}' \
+          --service registry \
+          --device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
+
+        with_backoff balena env add REGISTRY2_TOKEN_AUTH_REALM 'https://api.${{ env.SUBDOMAIN }}.${{ inputs.dns_tld }}/auth/v1/token' \
+          --service registry \
+          --device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
+
+        with_backoff balena env add REGISTRY2_S3_REGION_ENDPOINT 's3.${{ env.SUBDOMAIN }}.${{ inputs.dns_tld }}' \
+          --device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
+
+        with_backoff balena env add WEBRESOURCES_S3_HOST 's3.${{ env.SUBDOMAIN }}.${{ inputs.dns_tld }}' \
+          --device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
+
+        # https://github.com/balena-io/cert-manager/blob/master/entry.sh#L255-L278
+        # cert-manager will restore the last wildcard certificate from AWS/S3 to avoid
+        # being rate limited by LetsEncrypt/ACME
+        with_backoff balena env add AWS_S3_BUCKET '${{ env.AWS_S3_CERTS_BUCKET }}' \
+          --service cert-manager \
+          --device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
+
+        # FIXME: still required?
+        with_backoff balena env add COMMON_REGION '${{ env.AWS_REGION }}' \
+          --device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
+
+        with_backoff balena env add SUPERUSER_EMAIL 'admin@${{ env.SUBDOMAIN }}.${{ inputs.dns_tld }}' \
+          --device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
+
+        with_backoff balena env add ORG_UNIT openBalena \
+          --device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
+
+
+        # unstable/unsupported functionality
+        with_backoff balena env add HIDE_UNVERSIONED_ENDPOINT 'false' \
+          --service api \
+          --device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
+
+        with_backoff balena env add RELEASE_ASSETS_TEST 'true' \
+          --service sut \
+          --device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
+
+    - name: configure test device secrets
+      if: ${{ github.event_name == 'pull_request' && github.event.action != 'closed'}}
+      run: |
+        set -ue
+
+        [[ '${{ vars.VERBOSE || 'false' }}' =~ on|On|Yes|yes|true|True ]] && set -x
+
+        source src/balena-tests/functions
+
+        with_backoff balena login --token '${{ secrets.BALENA_API_KEY }}'
+
+        # cert-manager requires it to get whoami information for the user
+        with_backoff balena env add API_TOKEN '${{ secrets.BALENA_API_KEY }}' \
+          --service cert-manager \
+          --device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
+
+        # cert-manager requires is to request wildcard SSL certificate from LetsEncrypt
+        with_backoff balena env add CLOUDFLARE_API_TOKEN '${{ secrets.CLOUDFLARE_API_TOKEN }}' \
+          --service cert-manager \
+          --device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
+
+        # AWS credentials to backup/restore PKI assets
+        with_backoff balena env add AWS_ACCESS_KEY_ID '${{ env.AWS_ACCESS_KEY_ID }}' \
+          --service cert-manager \
+          --device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
+
+        with_backoff balena env add AWS_SECRET_ACCESS_KEY '${{ env.AWS_SECRET_ACCESS_KEY }}' \
+          --service cert-manager \
+          --device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
+
+    - name: provision ephemeral test device
+      id: provision-test-device
+      if: ${{ github.event_name == 'pull_request' && github.event.action != 'closed'}}
+      run: |
+        set -ue
+
+        [[ '${{ vars.VERBOSE || 'false' }}' =~ on|On|Yes|yes|true|True ]] && set -x
+
+        for subnet_id in ${{ env.AWS_VPC_SUBNET_IDS }}; do
+            # spot, on-demand
+            for market_type in ${{ vars.MARKET_TYPES || 'spot' }}; do
+                # https://docs.aws.amazon.com/cli/latest/reference/ec2/run-instances.html
+                response="$(aws ec2 run-instances \
+                  --launch-template 'LaunchTemplateId=${{ env.AWS_EC2_LAUNCH_TEMPLATE }},Version=${{ env.AWS_EC2_LT_VERSION }}' \
+                  --instance-type '${{ env.AWS_EC2_INSTANCE_TYPE }}' \
+                  $([[ $market_type =~ spot ]] && echo '--instance-market-options MarketType=spot') \
+                  --security-group-ids '${{ env.AWS_VPC_SECURITY_GROUP_IDS }}' \
+                  --subnet-id "${subnet_id}" \
+                  --associate-public-ip-address \
+                  --user-data file://config.json \
+                  --tag-specifications "ResourceType=instance,Tags=[{Key=Name,Value=balena-tests},{Key=MarketType,Value=${market_type}},{Key=Owner,Value=${{ env.AWS_IAM_USERNAME }}},{Key=GITHUB_SHA,Value=${GITHUB_SHA}-tests}]" || true)"
+
+                [[ -n $response ]] && break
+            done
+            [[ -n $response ]] && break
+        done
+
+        [[ -z $response ]] && exit 1
+
+        instance_id="$(echo "${response}" | jq -r '.Instances[].InstanceId')"
+
+        aws ec2 wait instance-running --instance-ids "${instance_id}"
+
+        aws ec2 wait instance-status-ok --instance-ids "${instance_id}"
+
+        echo "instance_id=${instance_id}" >> "${GITHUB_OUTPUT}"
+
+      env:
+        AWS_ACCESS_KEY_ID: ${{ env.AWS_ACCESS_KEY_ID }}
+        AWS_SECRET_ACCESS_KEY: ${{ env.AWS_SECRET_ACCESS_KEY }}
+        AWS_DEFAULT_REGION: ${{ env.AWS_REGION }}
+
+    - name: provision SSH key
+      id: provision-ssh-key
+      # wait for cloud-config
+      # https://github.com/balena-os/cloud-config
+      timeout-minutes: 5
+      if: ${{ github.event_name == 'pull_request' && github.event.action != 'closed'}}
+      run: |
+        set -ue
+
+        [[ '${{ vars.VERBOSE || 'false' }}' =~ on|On|Yes|yes|true|True ]] && set -x
+
+        source src/balena-tests/functions
+
+        with_backoff balena login --token '${{ secrets.BALENA_API_KEY }}'
+
+        if ! [[ -e "${HOME}/.ssh/id_rsa" ]]; then
+            ssh-keygen -N '' \
+              -C "$(balena whoami | grep EMAIL | cut -c11-)" \
+              -f "${HOME}/.ssh/id_rsa"
+        fi
+
+        echo "::notice::check $(balena keys | wc -l) keys"
+
+        match=''
+        for key in $(balena keys | grep -v ID | awk '{print $1}'); do
+            fp=$(balena key ${key} | tail -n 1 | ssh-keygen -E md5 -lf /dev/stdin | awk '{print $2}')
+            if [[ $fp =~ $(ssh-keygen -E md5 -lf "${HOME}/.ssh/id_rsa" | awk '{print $2}') ]]; then
+                match="${key}"
+                break
+            fi
+        done
+
+        if [[ -z $match ]]; then
+            balena key add "${GITHUB_SHA}" "${HOME}/.ssh/id_rsa.pub"
+        else
+            balena keys
+        fi
+
+        pgrep ssh-agent || ssh-agent -a "${SSH_AUTH_SOCK}"
+
+        ssh-add "${HOME}/.ssh/id_rsa"
+
+        while ! [[ "$(ssh-uuid -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
+          ${{ steps.register-test-device.outputs.balena_device_uuid }}.balena \
+          cat /mnt/boot/config.json | jq -r .uuid)" =~ ${{ steps.register-test-device.outputs.balena_device_uuid }} ]]; do
+
+            echo "::warning::Still working..."
+            sleep "$(( (RANDOM % 5) + 5 ))s"
+        done
+
+        echo "key_id=${GITHUB_SHA}" >> "${GITHUB_OUTPUT}"
+
+      env:
+        SSH_AUTH_SOCK: /tmp/ssh_agent.sock
+
+    - name: wait for application
+      timeout-minutes: 10
+      if: ${{ github.event_name == 'pull_request' && github.event.action != 'closed'}}
+      run: |
+        set -ue
+
+        [[ '${{ vars.VERBOSE || 'false' }}' =~ on|On|Yes|yes|true|True ]] && set -x
+
+        source src/balena-tests/functions
+
+        with_backoff balena login --token '${{ secrets.BALENA_API_KEY }}'
+
+        balena whoami && ssh-add -l
+
+        while [[ "$(curl -X POST --silent --retry ${{ env.RETRY }} --fail \
+          'https://api.${{ inputs.environment }}/supervisor/v1/device' \
+          --header 'authorization: Bearer ${{ secrets.BALENA_API_KEY }}' \
+          --header 'Content-Type:application/json' \
+          --data '{"uuid": "${{ steps.register-test-device.outputs.balena_device_uuid }}", "method": "GET"}' \
+          --compressed | jq -r '.update_pending')" =~ ^true$ ]]; do
+
+            sleep "$(( ( RANDOM % ${{ env.RETRY }} ) + ${{ env.RETRY }} ))s"
+        done
+
+        # wait for services to start running
+        while with_backoff ssh-uuid -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
+          ${{ steps.register-test-device.outputs.balena_device_uuid }}.balena \
+          'balena ps -q | xargs balena inspect | jq -r .[].State.Status' \
+          | grep -E 'created|restarting|removing|paused|exited|dead'; do
+
+            echo "::warning::Still working..."
+            sleep "$(( (RANDOM % 30) + 30 ))s"
+        done
+
+        # wait for Docker healthchecks
+        while with_backoff ssh-uuid -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
+          ${{ steps.register-test-device.outputs.balena_device_uuid }}.balena \
+          'balena ps -q | xargs balena inspect \
+          | jq -r ".[] | select(.State.Health.Status!=null).Name + \":\" + .State.Health.Status"' \
+          | grep -E ':starting|:unhealthy'; do
+
+            echo "::warning::Still working..."
+            sleep "$(( (RANDOM % 30) + 30 ))s"
+        done
+
+      env:
+        SSH_AUTH_SOCK: /tmp/ssh_agent.sock
+
+    # (TBC) https://www.balena.io/docs/reference/supervisor/docker-compose/
+    # due to lack of long form depends_on support in compositions, restart to ensure all
+    # components are running with the latest configuration; preferred over restart via
+    # Supervisor API restart due to potential HTTP [timeouts](https://github.com/balena-os/balena-supervisor/issues/1157)
+    - name: restart components
+      timeout-minutes: 10
+      if: ${{ github.event_name == 'pull_request' && github.event.action != 'closed'}}
+      run: |
+        set -ue
+
+        [[ '${{ vars.VERBOSE || 'false' }}' =~ on|On|Yes|yes|true|True ]] && set -x
+
+        source src/balena-tests/functions
+
+        with_backoff balena login --token '${{ secrets.BALENA_API_KEY }}'
+
+        balena whoami && ssh-add -l
+
+        with_backoff ssh-uuid -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
+          ${{ steps.register-test-device.outputs.balena_device_uuid }}.balena \
+          "balena ps -aq | xargs balena inspect \
+          | jq -re '.[]
+          | select(.Name | contains(\"_supervisor\") | not).Id' \
+          | xargs balena restart"
+
+        # wait for Docker healthchecks
+        while with_backoff ssh-uuid -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
+          ${{ steps.register-test-device.outputs.balena_device_uuid }}.balena \
+          'balena ps -q | xargs balena inspect \
+          | jq -r ".[] | select(.State.Health.Status!=null).Name + \":\" + .State.Health.Status"' \
+          | grep -E ':starting|:unhealthy'; do
+
+            echo "::warning::Still working..."
+            sleep "$(( (RANDOM % 30) + 30 ))s"
+        done
+
+      env:
+        SSH_AUTH_SOCK: /tmp/ssh_agent.sock
+
+    - name: SUT&DUT
+      if: ${{ github.event_name == 'pull_request' && github.event.action != 'closed'}}
+      timeout-minutes: 20
+      # https://giters.com/gfx/example-github-actions-with-tty
+      # https://github.com/actions/runner/issues/241#issuecomment-924327172
+      shell: 'script -q -e -c "bash {0}"'
+      run: |
+        set -ue
+
+        [[ '${{ vars.VERBOSE || 'false' }}' =~ on|On|Yes|yes|true|True ]] && set -x
+
+        source src/balena-tests/functions
+
+        with_backoff balena login --token '${{ secrets.BALENA_API_KEY }}'
+
+        balena whoami && ssh-add -l
+
+        (with_backoff ssh-uuid -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
+          ${{ steps.register-test-device.outputs.balena_device_uuid }}.balena \
+          "balena ps -aq | xargs balena inspect \
+          | jq -re '.[] | select(.Name | contains(\"sut_\")).Id' \
+          | xargs balena logs -f") &
+
+        # tests service is working while its status == running
+        status=''
+        while [[ "$status" =~ Running ]]; do
+            status="$(curl --silent --retry ${{ env.RETRY }} --fail \
+              'https://api.${{ inputs.environment }}/supervisor/v2/applications/state' \
+              --header 'authorization: Bearer ${{ secrets.BALENA_API_KEY }}' \
+              --header 'Content-Type:application/json' \
+              --data '{"uuid": "${{ steps.register-test-device.outputs.balena_device_uuid }}", "method": "GET"}' \
+              --compressed | jq -r '.[].services.sut.status')"
+
+            echo "::warning::Still working..."
+            sleep "$(( ( RANDOM % ${{ env.RETRY }} ) + ${{ env.RETRY }} ))s"
+        done
+
+        # .. once the service exits with status == exited, it is assumed to be finished
+        status=''
+        while ! [[ "$status" =~ exited ]]; do
+            echo "::warning::Still working..."
+            status="$(curl --silent --retry ${{ env.RETRY }} --fail \
+              'https://api.${{ inputs.environment }}/supervisor/v2/applications/state' \
+              --header 'authorization: Bearer ${{ secrets.BALENA_API_KEY }}' \
+              --header 'Content-Type:application/json' \
+              --data '{"uuid": "${{ steps.register-test-device.outputs.balena_device_uuid }}", "method": "GET"}' \
+              --compressed | jq -r '.[].services.sut.status')"
+
+            sleep "$(( ( RANDOM % ${{ env.RETRY }} ) + ${{ env.RETRY }} ))s"
+        done
+
+        # .. check its exit code
+        expected_exit_code=0
+        actual_exit_code="$(with_backoff ssh-uuid -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
+          ${{ steps.register-test-device.outputs.balena_device_uuid }}.balena \
+          "balena ps -aq | xargs balena inspect \
+          | jq -re '.[] | select(.Name | contains(\"sut_\")).State.ExitCode'")"
+
+        [[ $expected_exit_code -eq $actual_exit_code ]] || false
+
+      env:
+        SSH_AUTH_SOCK: /tmp/ssh_agent.sock
+        ATTEMPTS: 2
+
+    - name: remove SSH key
+      if: always()
+      continue-on-error: true
+      run: |
+        set -ue
+
+        [[ '${{ vars.VERBOSE || 'false' }}' =~ on|On|Yes|yes|true|True ]] && set -x
+
+        source src/balena-tests/functions
+
+        with_backoff balena login --token '${{ secrets.BALENA_API_KEY }}'
+
+        with_backoff balena keys | grep ${{ steps.provision-ssh-key.outputs.key_id }} \
+          | awk '{print $1}' | xargs balena key rm --yes
+
+        pgrep ssh-agent && (pgrep ssh-agent | xargs kill)
+
+        rm -f /tmp/ssh_agent.sock
+
+    - name: destroy balena test device
+      if: always()
+      continue-on-error: true
+      run: |
+        set -ue
+
+        [[ '${{ vars.VERBOSE || 'false' }}' =~ on|On|Yes|yes|true|True ]] && set -x
+
+        source src/balena-tests/functions
+
+        with_backoff balena login --token '${{ secrets.BALENA_API_KEY }}'
+
+        with_backoff balena device rm ${{ steps.register-test-device.outputs.balena_device_uuid }} --yes
+
+      env:
+        AWS_ACCESS_KEY_ID: ${{ env.AWS_ACCESS_KEY_ID }}
+        AWS_SECRET_ACCESS_KEY: ${{ env.AWS_SECRET_ACCESS_KEY }}
+        AWS_DEFAULT_REGION: ${{ env.AWS_REGION }}
+
+    # always destroy test EC2 instances even if the workflow is cancelled
+    - name: destroy AWS test device
+      if: always()
+      run: |
+        set -ue
+
+        [[ '${{ vars.VERBOSE || 'false' }}' =~ on|On|Yes|yes|true|True ]] && set -x
+
+        source src/balena-tests/functions
+
+        if [[ -n '${{ steps.provision-test-device.outputs.instance_id }}' ]]; then
+            with_backoff aws ec2 terminate-instances \
+              --instance-ids ${{ steps.provision-test-device.outputs.instance_id }}
+        fi
+
+        with_backoff aws ec2 describe-instances --filters Name=tag:GITHUB_SHA,Values=${GITHUB_SHA}-tests \
+          | jq -r .Reservations[].Instances[].InstanceId \
+          | xargs aws ec2 terminate-instances --instance-ids
+
+        stale_instances=$(mktemp)
+        aws ec2 describe-instances --filters \
+          Name=tag:Name,Values=balena-tests \
+          Name=instance-state-name,Values=running \
+          | jq -re '.Reservations[].Instances[].InstanceId + " " + .Reservations[].Instances[].LaunchTime' > ${stale_instances} || true
+
+        if test -s "${stale_instances}"; then
+            while IFS= read -r line; do
+                instance_id=$(echo ${line} | awk '{print $1}')
+                launch_time=$(echo ${line} | awk '{print $2}')
+                now=$(date +%s)
+                then=$(date --date ${launch_time} +%s)
+                days_since_launch=$(( (now - then) / 86400 ))
+                if [[ -n $days_since_launch ]] && [[ $days_since_launch -ge 1 ]]; then
+                    with_backoff aws ec2 terminate-instances --instance-ids ${instance_id}
+                fi
+            done <${stale_instances}
+            rm -f ${stale_instances}
+        fi
+
+      env:
+        AWS_ACCESS_KEY_ID: ${{ env.AWS_ACCESS_KEY_ID }}
+        AWS_SECRET_ACCESS_KEY: ${{ env.AWS_SECRET_ACCESS_KEY }}
+        AWS_DEFAULT_REGION: ${{ env.AWS_REGION }}
+
+    # remove orphaned ACME DNS-01 validation records
+    # https://letsencrypt.org/docs/challenge-types/#dns-01-challenge
+    # FIXME: clean up older _acme-challenge.auto TXT records
+    - name: cleanup-dns-records
+      if: always()
+      continue-on-error: true
+      run: |
+        set -ue
+
+        [[ '${{ vars.VERBOSE || 'false' }}' =~ on|On|Yes|yes|true|True ]] && set -x
+
+        if [[ -n '${{ steps.register-test-device.outputs.balena_device_uuid }}' ]]; then
+            match="${{ steps.register-test-device.outputs.balena_device_uuid }}.${{ env.SUBDOMAIN }}"
+
+            zone_id="$(curl --silent --retry ${{ env.RETRY }} \
+              "https://api.cloudflare.com/client/v4/zones?name=${{ inputs.dns_tld }}" \
+              -H 'Authorization: Bearer ${{ secrets.CLOUDFLARE_API_TOKEN }}' | jq -r '.result[].id')"
+
+            for record in "$(curl --silent --retry ${{ env.RETRY }} \
+              "https://api.cloudflare.com/client/v4/zones/${zone_id}/dns_records" \
+              -H 'Authorization: Bearer ${{ secrets.CLOUDFLARE_API_TOKEN }}' \
+              | jq -r --arg match "${match}" '.result[] | select(((.type=="TXT") and (.name | contains($match))))' \
+              | base64)"; do
+
+                json="$(echo "${record}" | base64 -d | jq -r)"
+                id="$(echo "${json}" | jq -r .id)"
+                name="$(echo "${json}" | jq -r .name)"
+
+                if [[ -n $id ]] && [[ -n $name ]]; then
+                    echo "::warning::Orphaned DNS record ${name} (${id})..."
+
+                    if [[ -z $DRY_RUN ]]; then
+                        curl -X DELETE --silent --retry ${{ env.RETRY }} \
+                          "https://api.cloudflare.com/client/v4/zones/${zone_id}/dns_records/${id}" \
+                          -H 'Authorization: Bearer ${{ secrets.CLOUDFLARE_API_TOKEN }}'
+                    fi
+                fi
+            done
+        fi
+
+      env:
+        DRY_RUN: true

.gitignore

@@ -1,7 +1,3 @@
 .DS_Store
-.project
-.vagrant/
-
-/config
-/docker-compose.yml
-/package-lock.json
+.balena
+**/.env

.openbalenarc

@@ -1,50 +0,0 @@
-#!/bin/bash
-
-alias dc="/home/vagrant/openbalena/scripts/compose"
-
-function enter () {
-	if [[ $# -lt 1 ]]; then			
-		echo "Usage: enter <service name> [command]"
-		echo "  "
-		echo "  Runs a [command] in the service specified."
-		echo "  "
-		echo "  command:"
-		echo "    (default) /bin/bash"
-		echo "  "
-		echo "  example:"
-		echo "    enter api          # this will run the command '/bin/bash' in the API service, providing a shell prompt"
-		echo "    enter api uptime   # this will run the command 'uptime' in the API service, and return"
-		return 1
-	fi
-
-
-	service="$1"
-	shift
-	COMMAND=/bin/bash
-	if [[ $# -gt 0 ]]; then
-		COMMAND="$@"
-	fi
-	dc exec ${service} /bin/bash -c "${COMMAND}"
-}
-
-function logs () {
-	if [[ $# -lt 1 ]]; then			
-		echo "Usage: logs <service name> [options]"
-		echo "  "
-		echo "  Shows the logs from journalctl in the service specified."
-		echo "  "
-		echo "  options:"
-		echo "    -f  tail the log stream"
-		echo "    -n  number of lines to take"
-		echo "  "
-		echo "  example:"
-		echo "    logs api -fn100 # this will tail the API log, starting with the last 100 lines"
-		return 1
-	fi
-
-	service="$1"
-	shift
-	enter ${service} journalctl "$@"
-}
-
-cd /home/vagrant/openbalena

Makefile

@@ -1,4 +1,135 @@
-.PHONY: lint
+SHELL := bash
 
-lint:
-	shellcheck scripts/*
+# export all variables to child processes by default
+export
+
+# Include the .env file
+include .env
+
+DNS_TLD ?= $(error DNS_TLD not set)
+TMPKI := $(shell mktemp)
+STAGING_PKI ?= /usr/local/share/ca-certificates
+PRODUCTION_MODE ?= true
+ORG_UNIT ?= openBalena
+SUPERUSER_EMAIL ?= admin@$(DNS_TLD)
+
+.NOTPARALLEL: $(DOCKERCOMPOSE)
+
+.PHONY: help
+help: ## Print help message
+	@echo -e "$$(grep -hE '^\S+:.*##' $(MAKEFILE_LIST) | sed -e 's/:.*##\s*/:/' -e 's/^\(.\+\):\(.*\)/\\x1b[36m\1\\x1b[m:\2/' | column -c2 -t -s :)"
+
+.PHONY: lint
+lint: ## Lint shell scripts with shellcheck
+	find . -type f -name *.sh | xargs shellcheck
+
+.PHONY: verify
+verify: ## Ping the public API endpoint
+	curl --fail --retry 3 https://api.$(DNS_TLD)/ping
+	@printf '\n'
+
+# Write all supported variables to .env, whether they have been provided or not.
+# If they already exist in the .env they will be retained.
+# The existing .env takes priority over envs provided from the command line.
+.PHONY: config
+config: ## Rewrite the .env config from current context (env vars + env args + existing .env)
+ifneq ($(CLOUDFLARE_API_TOKEN),)
+ifneq ($(GANDI_API_TOKEN),)
+	$(error "CLOUDFLARE_API_TOKEN and GANDI_API_TOKEN cannot both be set")
+endif
+endif
+	@rm -f .env
+	@echo "DNS_TLD=$(DNS_TLD)" >> .env
+	@echo "ORG_UNIT=$(ORG_UNIT)" >> .env
+	@echo "SUPERUSER_EMAIL=$(SUPERUSER_EMAIL)" >> .env
+	@echo "PRODUCTION_MODE=$(PRODUCTION_MODE)" >> .env
+	@echo "GANDI_API_TOKEN=$(GANDI_API_TOKEN)" >> .env
+	@echo "CLOUDFLARE_API_TOKEN=$(CLOUDFLARE_API_TOKEN)" >> .env
+	@echo "ACME_EMAIL=$(ACME_EMAIL)" >> .env
+	@echo "HAPROXY_CRT=$(HAPROXY_CRT)" >> .env
+	@echo "HAPROXY_KEY=$(HAPROXY_KEY)" >> .env
+	@echo "ROOT_CA=$(ROOT_CA)" >> .env
+	@$(MAKE) showenv
+
+.PHONY: up
+up: config ## Start all services
+	@docker compose up --build -d
+	@until [[ $$(docker compose ps api --format json | jq -r '.Health') =~ healthy ]]; do printf '.'; sleep 3; done
+	@printf '\n'
+	@$(MAKE) showenv
+	@$(MAKE) showpass
+
+.PHONY: showenv
+showenv: ## Print the current contents of the .env config
+	@cat <.env
+	@printf '\n'
+
+.PHONY: printenv
+printenv: ## Print the current environment variables
+	@printenv
+
+.PHONY: showpass
+showpass: ## Print the superuser password
+	@docker compose exec api cat config/env | grep SUPERUSER_PASSWORD
+	@printf '\n'
+
+.PHONY: down
+down: ## Stop all services
+	@docker compose stop
+
+.PHONY: stop
+stop: down ## Alias for 'make down'
+
+.PHONY: restart
+restart: ## Restart all services
+	@docker compose restart
+
+.PHONY: update
+update: # Pull and deploy latest changes from git
+	@git pull
+	@$(MAKE) up
+
+.PHONY: destroy ## Stop and remove any existing containers and volumes
+destroy:
+	@docker compose down --volumes --remove-orphans
+
+.PHONY: clean
+clean: destroy ## Alias for 'make destroy'
+
+.PHONY: self-signed
+self-signed: ## Install self-signed CA certificates
+	@sudo mkdir -p .balena $(STAGING_PKI)
+
+	@true | openssl s_client -showcerts -connect api.$(DNS_TLD):443 \
+	  | awk '/BEGIN CERTIFICATE/,/END CERTIFICATE/ {print $0}' > $(TMPKI).ca
+
+	@cat <$(TMPKI).ca | openssl x509 -text \
+	  | awk '/BEGIN CERTIFICATE/,/END CERTIFICATE/ {print $0}' > $(TMPKI).srv
+
+	@diff --suppress-common-lines --unchanged-line-format= \
+	  $(TMPKI).srv \
+	  $(TMPKI).ca | sudo tee $(STAGING_PKI)/ca-$(DNS_TLD).crt || true
+
+	@sudo update-ca-certificates
+	@cat <$(STAGING_PKI)/ca-$(DNS_TLD).crt | sudo tee .balena/ca-$(DNS_TLD).pem
+
+# FIXME: refactor this function to use 'make up'
+.PHONY: auto-pki
+auto-pki: config # Start all services using LetsEncrypt and ACME
+	@docker compose exec cert-manager rm -f /certs/export/chain.pem
+	@docker compose up -d
+	@until docker compose logs cert-manager | grep -Eq "/certs/export/chain.pem Certificate will not expire in [0-9] days"; do printf '.'; sleep 3; done
+	@until docker compose logs cert-manager | grep -q "subject=CN = ${DNS_TLD}"; do printf '.'; sleep 3; done
+	@until docker compose logs cert-manager | grep -q "issuer=C = US, O = Let's Encrypt, CN = R3"; do printf '.'; sleep 3; done
+	@until [[ $$(docker compose ps haproxy --format json | jq -r '.Health') =~ healthy ]]; do printf '.'; sleep 3; done
+	@printf '\n'
+	@$(MAKE) showenv
+	@$(MAKE) showpass
+
+.PHONY: pki-custom
+pki-custom: up ## Alias for 'make up'
+
+.PHONY: deploy
+deploy: up ## Alias for 'make up'
+
+.DEFAULT_GOAL = help

README.md

@@ -1,6 +1,8 @@
-<img alt="openBalena" src="docs/assets/openbalena-logo.svg" height="82">
+[![Flowzone](https://github.com/balena-io/open-balena/actions/workflows/flowzone.yml/badge.svg)](https://github.com/balena-io/open-balena/actions/workflows/flowzone.yml)
 
----
+![](./docs/images/openbalena-logo.svg)
+
+[![deploy button](https://balena.io/deploy.svg)](https://dashboard.balena-cloud.com/deploy?repoUrl=https://github.com/balena-io/open-balena)
 
 OpenBalena is a platform to deploy and manage connected devices. Devices run
 [balenaOS][balena-os-website], a host operating system designed for running
@@ -36,13 +38,17 @@ application to your device(s).
 
 The current release of openBalena has the following minimum version requirements:
 
-- balenaOS v2.58.3
-- balena CLI v12.38.5
+- balenaOS v5.2.8
+- balena CLI v18.2.2
 
 If you are updating from previous openBalena versions, ensure you update the balena
-CLI and reprovision any devices to at least the minimum required versions in order
+CLI and re-provision any devices to at least the minimum required versions in order
 for them to be fully compatible with this release, as some features may not work.
 
+While in-place openBalena upgrades may succeed, when performing major updates, it is
+recommended for a new instance to be deployed in parallel with the existing one, followed
+by copying state across and pointing a test device to the new instance.
+
 
 ## Documentation
 
@@ -103,25 +109,66 @@ improvements and new functionality is planned:
 
 ## Differences between openBalena and balenaCloud
 
-| openBalena                                          | balenaCloud                                                                                                                                                                                                                                                             |
-| -----                                               | ----                                                                                                                                                                                                                                                                    |
-| Device updates using full images                    | Device updates using [delta images](https://www.balena.io/docs/learn/deploy/delta/)                                                                                                                                                                                     |
-| Support for a single user                           | Support for [multiple users](https://www.balena.io/docs/learn/manage/account/#application-members)                                                                                                                                                                      |
-| Self-hosted deployment and scaling                  | balena-managed scaling and deployment |
-| Community support via [forums][forums]              | Private support on [paid plans](https://www.balena.io/pricing/)                                                                                                                                           |
-| Deploy via `balena deploy` only                     | Build remotely with native builders using [`balena push`](https://www.balena.io/docs/learn/deploy/deployment/#balena-push) or  [`git push`](https://www.balena.io/docs/learn/deploy/deployment/#git-push) |
-| No support for building via `git push`              | Use the same CI workflow with [`git push`](https://www.balena.io/docs/learn/deploy/deployment/#git-push)                                                                                                  |
-| No public URL support                               | Serve websites directly from device with [public device URLs](https://www.balena.io/docs/learn/manage/actions/#enable-public-device-url)                                                                  |
-| Management via `balena-cli` only                    | Cloud-based device management dashboard                                                                                                                                                                   |
-| Download images from [balena.io][balena-os-website] | Download preconfigured images directly from the dashboard                                                                                                                                                 |
-| No supported remote diagnostics                     | Remote device diagnostics                                                                                                                                                                                 |
+Whilst openBalena and balenaCloud share the same core technology, there are some key
+differences. First, openBalena is self-hosted, whereas balenaCloud is hosted by balena and
+therefore handles security, maintenance, scaling, and reliability of all the backend
+services. OpenBalena is also single user, whereas balenaCloud supports multiple users and
+organizations. OpenBalena also lacks some of the commercial features that define
+balenaCloud, such as the web-based dashboard and updates with binary container deltas.
 
-Additionally, refer back to the [roadmap](#roadmap) above for planned but not yet implemented features.
+The following table contains the main differences between both:
+
+| openBalena                                                                                 | balenaCloud                                                                                                                                                                                               |
+| ------------------------------------------------------------------------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| Device updates using full Docker images                                                    | Device updates using [delta images](https://www.balena.io/docs/learn/deploy/delta/)                                                                                                                       |
+| Support for a single user                                                                  | Support for [multiple users](https://www.balena.io/docs/learn/manage/account/#application-members)                                                                                                        |
+| Self-hosted deployment and scaling                                                         | balena-managed scaling and deployment                                                                                                                                                                     |
+| Community support via [forums][forums]                                                     | Private support on [paid plans](https://www.balena.io/pricing/)                                                                                                                                           |
+| Build locally and deploy via `balena-cli`                                                  | Build remotely with native builders using [`balena push`](https://www.balena.io/docs/learn/deploy/deployment/#balena-push) or  [`git push`](https://www.balena.io/docs/learn/deploy/deployment/#git-push) |
+| No public device URL support                                                               | Serve websites directly from device with [public device URLs](https://www.balena.io/docs/learn/manage/actions/#enable-public-device-url)                                                                  |
+| Management via `balena-cli` only                                                           | Cloud-based device management dashboard                                                                                                                                                                   |
+| Download images from [balena.io][balena-os-website] and configure locally via `balena-cli` | Download configured images directly from the dashboard                                                                                                                                                    |
+| No remote device diagnostics                                                               | Remote device diagnostics                                                                                                                                                                                 |
+
+Additionally, refer back to the [roadmap](#roadmap) above for planned but not yet
+implemented features.
 
 
 ## License
 
-OpenBalena is licensed under the terms of AGPL v3. See [LICENSE](LICENSE) for details.
+OpenBalena is licensed under the terms of AGPL v3. See [LICENSE] for details.
+
+
+## FAQ
+
+### How do you ensure continuity of openBalena? Are there security patches on openBalena?
+openBalena is an open source initiative which is mostly driven by us, but it also gets
+contributions from the community. We work to keep openBalena as up to date as our
+bandwidth allows, especially with security patches. That said, we do not have a policy or
+guarantee of a software release schedule. However, it is in our best interest to keep
+openBalena updated and patched since we also use it for balenaCloud.
+
+### How do you ensure the "Join" command actually works between openBalena and
+balenaCloud?
+The `balena join ..` command is frequently used for moving devices between openBalena,
+and balenaCloud environments. This command extends `balena os configure ..`, which is the
+basic tool balena uses for configuring devices.
+
+### Is it "production ready"?
+While we actually have some rather large fleets using openBalena, we consider it to be
+perpetually in "beta". This means potentially introducing breaking changes between
+releases.
+
+### Can new device type be added to openBalena?
+openBalena imports the following public [device-types] "out of the box". You can specify
+your own contracts repository by overriding `CONTRACTS_PUBLIC_REPO_NAME`,
+`CONTRACTS_PUBLIC_REPO_OWNER` and `IMAGE_STORAGE_BUCKET` environment variables on the API
+service/container.
+
+### Are there open-source UI dashboards from the community for openBalena?
+Yes! Here are a few:
+- [open-balena-admin / open-balena-ui](https://github.com/dcaputo-harmoni/open-balena-admin) by [dcaputo-harmoni](https://github.com/dcaputo-harmoni) who first posted about [here](https://forums.balena.io/t/open-balena-admin-an-admin-interface-for-openbalena/355324) in our Forums :)
+- [open-balena-dashboard](https://github.com/Razikus/open-balena-dashboard) by [Razikus](https://github.com/Razikus)
 
 
 [balena-cli]: https://github.com/balena-io/balena-cli
@@ -134,10 +181,14 @@ OpenBalena is licensed under the terms of AGPL v3. See [LICENSE](LICENSE) for de
 [forums]: https://forums.balena.io/c/open-balena
 [getting-started]: https://balena.io/open/docs/getting-started
 [issue-tracker]: https://github.com/balena-io/open-balena/issues
+[LICENSE]: https://github.com/balena-io/open-balena/blob/master/LICENSE
+[open-balena-admin / open-balena-ui]: https://github.com/dcaputo-harmoni/open-balena-admin
 [open-balena-api]: https://github.com/balena-io/open-balena-api
+[open-balena-dashboard]: https://github.com/Razikus/open-balena-dashboard
 [open-balena-db]: https://github.com/balena-io/open-balena-db
 [open-balena-registry]: https://github.com/balena-io/open-balena-registry
 [open-balena-s3]: https://github.com/balena-io/open-balena-s3
 [open-balena-vpn]: https://github.com/balena-io/open-balena-vpn
 [open-balena-website]: https://balena.io/open
 [pulls]: https://github.com/balena-io/open-balena/pulls
+[device-types]: https://github.com/balena-io/contracts/blob/master/contracts/hw.device-type

VERSION

@@ -1 +1 @@
-3.1.2
+4.0.3

Vagrantfile

@@ -1,42 +0,0 @@
-Vagrant.require_version '>= 2.0.0'
-
-[ 'vagrant-vbguest', 'vagrant-docker-compose' ].each do |p|
-  unless Vagrant.has_plugin?(p)
-    raise "Please install missing plugin: vagrant plugin install #{p}"
-  end
-end
-
-Vagrant.configure('2') do |config|
-  config.vm.define 'openbalena'
-  config.vm.hostname = 'openbalena-vagrant'
-  config.vm.box = 'bento/ubuntu-18.04'
-
-  config.vm.network "public_network",
-    use_dhcp_assigned_default_route: true
-
-  config.vm.synced_folder '.', '/vagrant', disabled: true
-  config.vm.synced_folder '.', '/home/vagrant/openbalena'
-
-  config.ssh.forward_agent = true
-
-  config.vm.provision :docker
-
-  $provision = <<-SCRIPT
-    DOCKER_COMPOSE_VERSION=1.24.0
-
-    touch /home/vagrant/.bashrc
-    grep -Fxq 'source /home/vagrant/openbalena/.openbalenarc' /home/vagrant/.bashrc || echo 'source /home/vagrant/openbalena/.openbalenarc' >> /home/vagrant/.bashrc
-
-    curl -o- https://raw.githubusercontent.com/creationix/nvm/v0.34.0/install.sh | bash
-    source "/home/vagrant/.nvm/nvm.sh" # This loads nvm
-    nvm install 10.15.0 && nvm use 10.15.0
-
-    # Install a newer version of docker-compose
-    (cd /usr/local/bin; \
-    sudo curl -o docker-compose --silent --location https://github.com/docker/compose/releases/download/$DOCKER_COMPOSE_VERSION/docker-compose-Linux-x86_64; \
-    sudo chmod a+x docker-compose)
-  SCRIPT
-
-  config.vm.provision :shell, privileged: false, inline: $provision
-
-end

balena.yml

@@ -0,0 +1,26 @@
+name: openBalena
+type: sw.application
+description: https://www.balena.io/open
+post-provisioning: |
+  [![Flowzone](https://github.com/balena-io/open-balena/actions/workflows/flowzone.yml/badge.svg)](https://github.com/balena-io/open-balena/actions/workflows/flowzone.yml)
+
+  ## Getting Started
+
+  * https://open-balena.pages.dev/#getting-started
+
+assets:
+  repository:
+    type: blob.asset
+    data:
+      url: 'https://github.com/balena-io/open-balena'
+  logo:
+    type: blob.asset
+    data:
+      url: 'https://raw.githubusercontent.com/balena-io/open-balena/master/logo.png'
+data:
+  defaultDeviceType: generic-amd64
+  supportedDeviceTypes:
+    - generic-amd64
+    - genericx86-64-ext
+    - intel-nuc
+version: 4.0.3

compose/common.yml

@@ -1,17 +0,0 @@
-version: "2.0"
-
-services:
-  component:
-    cap_add:
-      - SYS_ADMIN
-      - SYS_RESOURCE
-    environment:
-      - CONFD_BACKEND=ENV
-    tmpfs:
-      - /run
-      - /sys/fs/cgroup
-    privileged: true
-
-  system:
-    security_opt:
-      - seccomp:unconfined

compose/mdns.yml

@@ -1,31 +0,0 @@
-version: "2.0"
-
-services:
-  balena-mdns-publisher:
-    image: balena/balena-mdns-publisher:${OPENBALENA_MDNS_PUBLISHER_VERSION_TAG}
-    network_mode: "host"
-    cap_add:
-        - SYS_RESOURCE
-        - SYS_ADMIN
-    security_opt:
-        - apparmor:unconfined
-    tmpfs:
-        - /run
-        - /sys/fs/cgroup
-    # balenaOS - Required for host DBus comms. Not required for standalone Linux
-    labels:
-        io.balena.features.dbus: '1'
-        io.balena.features.supervisor-api: '1'
-    environment:
-        CONFD_BACKEND: ENV
-        # The name of the TLD to use. This *must* match certificates used for the rest of
-        # the resin backend (eg. that for BALENA_ROOT_CA if present).
-        MDNS_TLD: ${OPENBALENA_HOST_NAME}
-        # List of subdomains to advertise. This must include all required hosts.
-        MDNS_SUBDOMAINS: '["api", "db", "registry", "s3", "tunnel", "vpn"]'
-        # The expectation is the DBus socket to use is always at the following location.
-        DBUS_SESSION_BUS_ADDRESS: "unix:path=/host/run/dbus/system_bus_socket"
-        # Selects the interface used for incoming connections from the wider subnet.
-        # For NUCs, this is `eno1`. If running natively, pick the appropriate interface.
-        # Alternatively, keep the default commented out to autoselect.
-        #INTERFACE: "eno1"

compose/services.yml

@@ -1,188 +0,0 @@
-version: "2.0"
-
-volumes:
-  certs: {}
-  cert-provider: {}
-  db: {}
-  redis: {}
-  registry: {}
-  s3: {}
-
-services:
-  api:
-    extends:
-      file: ./common.yml
-      service: component
-    image: balena/open-balena-api:${OPENBALENA_API_VERSION_TAG}
-    depends_on:
-      - db
-      - s3
-      - redis
-    environment:
-      API_VPN_SERVICE_API_KEY: ${OPENBALENA_API_VPN_SERVICE_API_KEY}
-      BALENA_ROOT_CA: ${OPENBALENA_ROOT_CA}
-      COOKIE_SESSION_SECRET: ${OPENBALENA_COOKIE_SESSION_SECRET}
-      DB_HOST: db
-      DB_PASSWORD: docker
-      DB_PORT: 5432
-      DB_USER: docker
-      DELTA_HOST: delta.${OPENBALENA_HOST_NAME}
-      DEVICE_CONFIG_OPENVPN_CA: ${OPENBALENA_VPN_CA_CHAIN}
-      DEVICE_CONFIG_SSH_AUTHORIZED_KEYS: ${OPENBALENA_SSH_AUTHORIZED_KEYS}
-      HOST: api.${OPENBALENA_HOST_NAME}
-      IMAGE_MAKER_URL: img.${OPENBALENA_HOST_NAME}
-      IMAGE_STORAGE_BUCKET: resin-production-img-cloudformation
-      IMAGE_STORAGE_PREFIX: images
-      IMAGE_STORAGE_ENDPOINT: s3.amazonaws.com
-      JSON_WEB_TOKEN_EXPIRY_MINUTES: 10080
-      JSON_WEB_TOKEN_SECRET: ${OPENBALENA_JWT_SECRET}
-      MIXPANEL_TOKEN: __unused__
-      PRODUCTION_MODE: "${OPENBALENA_PRODUCTION_MODE}"
-      PUBNUB_PUBLISH_KEY: __unused__
-      PUBNUB_SUBSCRIBE_KEY: __unused__
-      REDIS_HOST: redis
-      REDIS_PORT: 6379
-      REGISTRY2_HOST: registry.${OPENBALENA_HOST_NAME}
-      REGISTRY_HOST: registry.${OPENBALENA_HOST_NAME}
-      SENTRY_DSN: ""
-      TOKEN_AUTH_BUILDER_TOKEN: ${OPENBALENA_TOKEN_AUTH_BUILDER_TOKEN}
-      TOKEN_AUTH_CERT_ISSUER: api.${OPENBALENA_HOST_NAME}
-      TOKEN_AUTH_CERT_KEY: ${OPENBALENA_TOKEN_AUTH_KEY}
-      TOKEN_AUTH_CERT_KID: ${OPENBALENA_TOKEN_AUTH_KID}
-      TOKEN_AUTH_CERT_PUB: ${OPENBALENA_TOKEN_AUTH_PUB}
-      TOKEN_AUTH_JWT_ALGO: "ES256"
-      VPN_HOST: vpn.${OPENBALENA_HOST_NAME}
-      VPN_PORT: 443
-      VPN_SERVICE_API_KEY: ${OPENBALENA_VPN_SERVICE_API_KEY}
-      SUPERUSER_EMAIL: ${OPENBALENA_SUPERUSER_EMAIL}
-      SUPERUSER_PASSWORD: ${OPENBALENA_SUPERUSER_PASSWORD}
-
-  registry:
-    extends:
-      file: ./common.yml
-      service: component
-    image: balena/open-balena-registry:${OPENBALENA_REGISTRY_VERSION_TAG}
-    depends_on:
-      - s3
-      - redis
-    volumes:
-      - registry:/data
-    environment:
-      API_TOKENAUTH_CRT: ${OPENBALENA_TOKEN_AUTH_PUB}
-      BALENA_REGISTRY2_HOST: registry.${OPENBALENA_HOST_NAME}
-      BALENA_ROOT_CA: ${OPENBALENA_ROOT_CA}
-      BALENA_TOKEN_AUTH_ISSUER: api.${OPENBALENA_HOST_NAME}
-      BALENA_TOKEN_AUTH_REALM: https://api.${OPENBALENA_HOST_NAME}/auth/v1/token
-      COMMON_REGION: ${OPENBALENA_S3_REGION}
-      REGISTRY2_CACHE_ENABLED: "false"
-      REGISTRY2_CACHE_ADDR: 127.0.0.1:6379
-      REGISTRY2_CACHE_DB: 0
-      REGISTRY2_CACHE_MAXMEMORY_MB: 1024 # megabytes
-      REGISTRY2_CACHE_MAXMEMORY_POLICY: allkeys-lru
-      REGISTRY2_S3_REGION_ENDPOINT: ${OPENBALENA_S3_ENDPOINT}
-      REGISTRY2_S3_BUCKET: ${OPENBALENA_REGISTRY2_S3_BUCKET}
-      REGISTRY2_S3_KEY: ${OPENBALENA_S3_ACCESS_KEY}
-      REGISTRY2_S3_SECRET: ${OPENBALENA_S3_SECRET_KEY}
-      REGISTRY2_SECRETKEY: ${OPENBALENA_REGISTRY_SECRET_KEY}
-      REGISTRY2_STORAGEPATH: /data
-
-  vpn:
-    extends:
-      file: ./common.yml
-      service: component
-    image: balena/open-balena-vpn:${OPENBALENA_VPN_VERSION_TAG}
-    depends_on:
-      - api
-    cap_add:
-      - NET_ADMIN
-    environment:
-      API_SERVICE_API_KEY: ${OPENBALENA_API_VPN_SERVICE_API_KEY}
-      BALENA_API_HOST: api.${OPENBALENA_HOST_NAME}
-      BALENA_ROOT_CA: ${OPENBALENA_ROOT_CA}
-      BALENA_VPN_PORT: 443
-      PRODUCTION_MODE: "${OPENBALENA_PRODUCTION_MODE}"
-      RESIN_VPN_GATEWAY: 10.2.0.1
-      SENTRY_DSN: ""
-      VPN_HAPROXY_USEPROXYPROTOCOL: "true"
-      VPN_OPENVPN_CA_CRT: ${OPENBALENA_VPN_CA}
-      VPN_OPENVPN_SERVER_CRT: ${OPENBALENA_VPN_SERVER_CRT}
-      VPN_OPENVPN_SERVER_DH: ${OPENBALENA_VPN_SERVER_DH}
-      VPN_OPENVPN_SERVER_KEY: ${OPENBALENA_VPN_SERVER_KEY}
-      VPN_SERVICE_API_KEY: ${OPENBALENA_VPN_SERVICE_API_KEY}
-
-  db:
-    extends:
-      file: ./common.yml
-      service: system
-    image: balena/open-balena-db:${OPENBALENA_DB_VERSION_TAG}
-    volumes:
-      - db:/var/lib/postgresql/data
-
-  s3:
-    extends:
-      file: ./common.yml
-      service: component
-    image: balena/open-balena-s3:${OPENBALENA_S3_VERSION_TAG}
-    volumes:
-      - s3:/export
-    environment:
-      S3_MINIO_ACCESS_KEY: ${OPENBALENA_S3_ACCESS_KEY}
-      S3_MINIO_SECRET_KEY: ${OPENBALENA_S3_SECRET_KEY}
-      BUCKETS: ${OPENBALENA_S3_BUCKETS}
-
-  redis:
-    extends:
-      file: ./common.yml
-      service: system
-    image: redis:alpine
-    volumes:
-      - redis:/data
-
-  haproxy:
-    extends:
-      file: ./common.yml
-      service: system
-    build: ../src/haproxy
-    depends_on:
-      - api
-      - cert-provider
-      - db
-      - s3
-      - redis
-      - registry
-      - vpn
-    ports:
-      - "80:80"
-      - "443:443"
-      - "3128:3128"
-    expose:
-      - "222"
-      - "5432"
-      - "6379"
-    networks:
-      default:
-        aliases:
-          - api.${OPENBALENA_HOST_NAME}
-          - registry.${OPENBALENA_HOST_NAME}
-          - vpn.${OPENBALENA_HOST_NAME}
-          - db.${OPENBALENA_HOST_NAME}
-          - s3.${OPENBALENA_HOST_NAME}
-          - redis.${OPENBALENA_HOST_NAME}
-          - tunnel.${OPENBALENA_HOST_NAME}
-    environment:
-      BALENA_HAPROXY_CRT: ${OPENBALENA_ROOT_CRT}
-      BALENA_HAPROXY_KEY: ${OPENBALENA_ROOT_KEY}
-      BALENA_ROOT_CA: ${OPENBALENA_ROOT_CA}
-      HAPROXY_HOSTNAME: ${OPENBALENA_HOST_NAME}
-    volumes:
-      - certs:/certs:ro
-
-  cert-provider:
-    build: ../src/cert-provider
-    volumes:
-      - certs:/certs
-      - cert-provider:/usr/src/app/certs
-    environment:
-      ACTIVE: ${OPENBALENA_ACME_CERT_ENABLED}
-      DOMAINS: "api.${OPENBALENA_HOST_NAME},registry.${OPENBALENA_HOST_NAME},s3.${OPENBALENA_HOST_NAME},vpn.${OPENBALENA_HOST_NAME},tunnel.${OPENBALENA_HOST_NAME}"
-      OUTPUT_PEM: /certs/open-balena.pem

compose/template.yml

@@ -1,10 +0,0 @@
-# Project-specific config.
-#
-# All paths must be defined relative to `compose/services.yml` regardless of
-# the location of this file, i.e. refer to `my-open-balena-checkout/somedir`
-# as `../somedir`. This is because of the way docker-compose handles paths
-# when specifying multiple configs and open-balena always specifying
-# `compose/services.yml` as the "base" config.
-#
-# You may view the effective config with `scripts/compose config`.
-version: "2.0"

compose/versions

@@ -1,6 +0,0 @@
-export OPENBALENA_API_VERSION_TAG=v0.109.2
-export OPENBALENA_DB_VERSION_TAG=v4.1.0
-export OPENBALENA_MDNS_PUBLISHER_VERSION_TAG=v1.7.9
-export OPENBALENA_REGISTRY_VERSION_TAG=v2.13.11
-export OPENBALENA_S3_VERSION_TAG=v2.9.9
-export OPENBALENA_VPN_VERSION_TAG=v9.16.1

docker-compose.yml

@@ -0,0 +1,360 @@
+---
+version: '2.4'
+
+volumes:
+  builder-certs-ca: {}
+  builder-certs-client: {}
+  builder-data: {}
+  cert-manager-data: {}
+  certs-data: {}
+  db-data: {}
+  pki-data: {}
+  redis-data: {}
+  resin-data: {}
+  s3-data: {}
+
+x-default-healthcheck: &default-healthcheck
+  test: /usr/src/app/docker-hc
+  interval: 45s
+  timeout: 15s
+  retries: 3
+
+x-default-environment: &default-environment
+  # FIXME: hardcoded https://github.com/balena-io/open-balena-db/blob/master/create-resin-db.sh#L4
+  DB_NAME: resin
+  # FIXME: hardcoded https://github.com/balena-io/open-balena-db/blob/master/Dockerfile#L3-L4
+  DB_PASSWORD: docker
+  DB_USER: docker
+  LOG_LEVEL: DEBUG
+  PRODUCTION_MODE: 'false'
+
+x-default-healthcheck-trait: &with-default-healthcheck
+  healthcheck:
+    <<: *default-healthcheck
+
+x-default-volumes-trait: &with-default-volumes
+  volumes:
+    - certs-data:/certs
+    - resin-data:/balena
+
+x-default-privileges-trait: &with-default-privileges
+  cap_add:
+    - SYS_ADMIN
+    - SYS_RESOURCE
+  security_opt:
+    - apparmor=unconfined
+  tmpfs:
+    - /run
+    - /sys/fs/cgroup
+
+x-extended-privileges-trait: &with-extended-privileges
+  security_opt:
+    - apparmor=unconfined
+    - seccomp=unconfined
+
+x-all-privileges-trait: &with-all-privileges
+  privileged: true
+  cap_add:
+    - ALL
+
+x-network-privileges-trait: &with-network-privileges
+  cap_add:
+    - NET_ADMIN
+    - SYS_ADMIN
+    - SYS_RESOURCE
+
+x-base-service-definition: &base-service
+  restart: unless-stopped
+  # for docker-compose only, no effect on balenaCloud
+  env_file:
+    - .env
+  tty: 'true'  # send syastemd logs from containers to stdout
+
+services:
+  # https://github.com/balena-io/open-balena-api
+  api:
+    <<: [
+      *base-service,
+      *with-default-healthcheck,
+      *with-default-privileges,
+      *with-default-volumes,
+    ]
+    image: balena/open-balena-api:v22.2.3
+    depends_on:
+      - db
+      - redis
+      - s3
+    environment:
+      <<: *default-environment
+      CONTRACTS_PUBLIC_REPO_NAME: contracts
+      CONTRACTS_PUBLIC_REPO_OWNER: balena-io
+      DB_GENERAL_REPLICA_MAX_USES: 1000
+      DB_GENERAL_REPLICA_PORT: 5432
+      DB_HOST: db
+      DB_PORT: 5432
+      DB_STATE_REPLICA_MAX_USES: 1000
+      DB_STATE_REPLICA_PORT: 5432
+      DB_USER: docker
+      HOSTS_CONFIG: API_HOST:api,DB_HOST:db,DELTA_HOST:delta,HOST:api,REDIS_HOST:redis,TOKEN_AUTH_CERT_ISSUER:api,VPN_HOST:cloudlink,REGISTRY2_HOST:registry2
+      IMAGE_STORAGE_BUCKET: resin-production-img-cloudformation
+      IMAGE_STORAGE_ENDPOINT: s3.amazonaws.com
+      IMAGE_STORAGE_PREFIX: images
+      JSON_WEB_TOKEN_EXPIRY_MINUTES: 10080
+      NUM_WORKERS: 1
+      OAUTH_CALLBACK_PROTOCOL: https
+      PORT: 80
+      REDIS_HOST: redis:6379
+      REDIS_IS_CLUSTER: 'false'
+      TOKEN_AUTH_JWT_ALGO: ES256
+      TOKENS_CONFIG: API_SERVICE_API_KEY:hex,AUTH_RESINOS_REGISTRY_CODE:hex,COOKIE_SESSION_SECRET:hex,JSON_WEB_TOKEN_SECRET:hex,MIXPANEL_TOKEN:hex,SUPERUSER_PASSWORD:hex,TOKEN_AUTH_BUILDER_TOKEN:hex,VPN_GUEST_API_KEY:hex,VPN_SERVICE_API_KEY:hex,API_VPN_SERVICE_API_KEY:API_SERVICE_API_KEY,REGISTRY2_TOKEN:TOKEN_AUTH_BUILDER_TOKEN
+      TRUST_PROXY: 172.16.0.0/12
+      VPN_PORT: 443
+      WEBRESOURCES_S3_BUCKET: web-resources
+      WEBRESOURCES_S3_REGION: "us-east-1"  # this is required for minio
+
+  # https://github.com/balena-io/open-balena-registry
+  registry:
+    <<: [
+      *base-service,
+      *with-default-healthcheck,
+      *with-default-privileges,
+    ]
+    image: balena/open-balena-registry:v2.39.55
+    volumes:
+      - certs-data:/certs
+      - resin-data:/balena
+    depends_on:
+      - redis
+      - s3
+    environment:
+      COMMON_REGION: open-balena
+      HOSTS_CONFIG: REGISTRY2_HOST:registry2,REGISTRY2_TOKEN_AUTH_ISSUER:api,REGISTRY2_TOKEN_AUTH_REALM:api
+      REGISTRY2_CACHE_ADDR: redis:6379
+      REGISTRY2_CACHE_DB: 1
+      REGISTRY2_CACHE_ENABLED: 'true'
+      REGISTRY2_S3_BUCKET: registry-data
+      REGISTRY2_STORAGEPATH: /data
+      TOKENS_CONFIG: REGISTRY2_SECRETKEY:hex
+
+  # https://github.com/balena-io/open-balena-vpn
+  vpn:
+    <<: [
+      *base-service,
+      *with-default-healthcheck,
+      *with-default-volumes,
+      # privileges in order from minimum to maximum
+      *with-network-privileges,
+      *with-default-privileges,
+    ]
+    image: balena/open-balena-vpn:v11.30.9
+    depends_on:
+      - api
+    environment:
+      HOSTS_CONFIG: VPN_HOST:cloudlink
+      TOKENS_CONFIG: ','
+      VPN_HAPROXY_USEPROXYPROTOCOL: 'true'
+      VPN_PORT: 443
+      # ensure correct service instance IP is registered with the API
+      VPN_SERVICE_REGISTER_INTERFACE: eth0
+
+  # https://github.com/balena-io/open-balena-db
+  db:
+    <<: *base-service
+    image: balena/open-balena-db:v5.2.2
+    volumes:
+      - db-data:/var/lib/postgresql/data
+    environment:
+      <<: *default-environment
+    healthcheck:
+      test: pg_isready -U "$${DB_USER}" -d "$${DB_NAME}"
+
+  # https://github.com/balena-io/open-balena-s3
+  s3:
+    <<: [
+      *base-service,
+      *with-default-healthcheck,
+      *with-default-privileges,
+    ]
+    image: balena/open-balena-s3:v2.28.42
+    volumes:
+      - s3-data:/export
+      - certs-data:/certs
+      - resin-data:/balena
+    environment:
+      BUCKETS: registry-data;web-resources
+      HOSTS_CONFIG: REGISTRY2_S3_REGION_ENDPOINT:s3,WEBRESOURCES_S3_HOST:s3
+      TOKENS_CONFIG: REGISTRY2_S3_KEY:hex,REGISTRY2_S3_SECRET:hex,S3_MINIO_ACCESS_KEY:REGISTRY2_S3_KEY,S3_MINIO_SECRET_KEY:REGISTRY2_S3_SECRET,WEBRESOURCES_S3_ACCESS_KEY:REGISTRY2_S3_KEY,WEBRESOURCES_S3_SECRET_KEY:REGISTRY2_S3_SECRET
+
+  # https://hub.docker.com/_/redis
+  redis:
+    <<: *base-service
+    # https://redis.io/blog/what-redis-license-change-means-for-our-managed-service-providers/
+    image: redis:7.2-alpine
+    volumes:
+      - redis-data:/data
+    healthcheck:
+      <<: *default-healthcheck
+      test: echo INFO | redis-cli | grep redis_version
+
+  # https://github.com/balena-io/open-balena-haproxy
+  haproxy:
+    <<: [
+      *base-service,
+      *with-default-privileges,
+      *with-default-volumes,
+    ]
+    build: src/haproxy
+    sysctls:
+      # https://github.com/docker-library/haproxy/issues/160
+      net.ipv4.ip_unprivileged_port_start: 0
+    healthcheck:
+      <<: *default-healthcheck
+      test: true | openssl s_client -connect localhost:443
+    ports:
+      # haproxy/http
+      - "80:80/tcp"
+      # haproxy/tcp-router
+      - "443:443/tcp"
+      # haproxy/stats
+      - "1936:1936/tcp"
+    environment:
+      LOGLEVEL: info
+
+  # dynamically configure Docker network aliases based on DNS_TLD and ALIAS list
+  # allows DNS resolution from systemd-less images on the Docker network
+  haproxy-sidecar:
+    <<: *base-service
+    build: src/haproxy-sidecar
+    volumes:
+      - /var/run/docker.sock:/host/run/docker.sock
+    environment:
+      DOCKER_HOST: unix:///host/run/docker.sock
+      # resolved internally as {{service}}.{{dns-tld-without-balena-device-uuid}} to haproxy service
+      ALIASES: api,ca,cloudlink,db,delta,logs,redis,registry2,s3,stats,tunnel
+    labels:
+      io.balena.features.balena-socket: 1
+      io.balena.features.supervisor-api	: 1
+
+  # https://github.com/balena-io/cert-manager
+  # https://certbot.eff.org/docs/using.html
+  # https://certbot-dns-cloudflare.readthedocs.io/
+  cert-manager:
+    <<: *base-service
+    build: src/cert-manager
+    volumes:
+      - cert-manager-data:/etc/letsencrypt
+      - certs-data:/certs
+      - resin-data:/balena
+    depends_on:
+      - balena-ca
+    environment:
+      # wildcard certificate for reverse proxy
+      SSH_KEY_NAMES: ','
+      SUBJECT_ALTERNATE_NAMES: '*'
+    labels:
+      io.balena.features.balena-api: 1
+      io.balena.features.supervisor-api: 1
+
+  # https://github.com/balena-io/ca-private
+  # https://github.com/cloudflare/cfssl/blob/master/doc/api/intro.txt
+  balena-ca:
+    <<: *base-service
+    image: balena/ca-private:v0.0.14
+    volumes:
+      - pki-data:/pki
+      - certs-data:/certs
+      - resin-data:/balena
+    healthcheck:
+      test: curl --silent -I --fail localhost:8888
+      interval: 60s
+      timeout: 60s
+      retries: 10
+    labels:
+      # future expansion
+      io.balena.features.balena-api: 1
+      io.balena.features.supervisor-api: 1
+
+
+
+  # --- the following are not required for runtime operation of openBalena
+
+  # only relevant when running in AWS/EC2
+  tag-sidecar:
+    build: src/tag-sidecar
+    restart: no
+    environment:
+      ENABLED: 'true'
+    labels:
+      io.balena.features.balena-api: 1
+
+  # Software Under Test (SUT) tests orchestrator
+  sut:
+    <<: [
+      *base-service,
+      *with-extended-privileges,
+      *with-network-privileges,
+    ]
+    build: src/balena-tests
+    command: /usr/sbin/balena.sh
+    environment:
+      DOCKER_CERT_PATH: /docker-pki/client
+      DOCKER_HOST: docker:2376
+      DOCKER_TLS_VERIFY: 'true'
+      GUEST_IMAGE: /balena/balena.img
+    volumes:
+      - builder-certs-client:/docker-pki/client
+      - certs-data:/certs
+      - resin-data:/balena
+    labels:
+      io.balena.features.balena-api: 1
+      io.balena.features.supervisor-api: 1
+    restart: no
+
+  # virtual Device Under Test (DUT)
+  dut:
+    <<: [
+      *base-service,
+      *with-extended-privileges,
+      *with-network-privileges,
+    ]
+    # https://hub.docker.com/r/qemux/qemu-docker
+    # https://github.com/qemus/qemu-docker
+    build: src/test-device
+    entrypoint:
+      - /bin/sh
+      - -c
+    command:
+      - /usr/sbin/balena.sh
+    environment:
+      GUEST_IMAGE: /balena/balena.img
+      MEMORY: 3072M
+      CPU: 4
+    volumes:
+      - resin-data:/balena
+    devices:
+      - /dev/net/tun
+    restart: no
+
+  # https://hub.docker.com/_/docker
+  # pseudo(builder) service for balena-tests
+  docker:
+    <<: [
+      *base-service,
+      *with-extended-privileges,
+      *with-network-privileges,
+    ]
+    image: docker:dind
+    volumes:
+      - builder-data:/var/lib/docker
+      - builder-certs-ca:/docker-pki/ca
+      - builder-certs-client:/docker-pki/client
+      - /sys:/sys
+    environment:
+      DOCKER_TLS_CERTDIR: /docker-pki
+    healthcheck:
+      test: docker system info
+      interval: 60s
+      timeout: 60s
+      retries: 5
+    labels:
+      io.balena.features.sysfs: 1

docs/getting-started.md

@@ -0,0 +1,485 @@
+# openBalena Getting Started Guide
+
+This guide will walk you through the steps of deploying an openBalena server, that
+together with the balena CLI, will enable you to create and manage a fleet of devices
+running on your own infrastructure, on premises or in the cloud. The openBalena servers
+must be reachable by the devices, which is easiest to achieve with cloud providers like
+AWS, Google Cloud, Digital Ocean and others.
+
+This guide assumes a setup with two separate machines:
+
+- A _server_, running Linux with at least 2GB of memory. These instructions were tested
+  with Ubuntu 20.04, 22.04 and 24.04 x64 servers. The server must have a working
+  installation of [Docker Engine] and you must have root permissions.
+- A _local machine_, running Linux, Windows or macOS where the balena CLI runs (as a
+  client to the openBalena server). The local machine must also have a working
+  installation of [Docker] so that application images can be built and deployed to your
+  device. It is also possible to use [balenaEngine] on a [balenaOS] device instead of
+  Docker.
+
+Additionally, a _device type_ and compatible flash media supported by [balenaOS]
+(e.g. Raspberry Pi) are required to complete the provisioning demo. Ensure the correct
+power supply is available to power this device.
+
+## Domain Configuration
+
+The following DNS records must be configured to point to the openBalena server prior to
+configuration:
+
+```text
+api.mydomain.com
+ca.mydomain.com
+cloudlink.mydomain.com
+logs.mydomain.com
+ocsp.mydomain.com
+registry2.mydomain.com
+s3.mydomain.com
+tunnel.mydomain.com
+```
+
+Alternatively you may consider adding a single wildcard DNS record `*.mydomain.com`.
+
+Check with your Internet domain name registrar for instructions on how to obtain a domain
+name and configure records.
+
+## Install openBalena on the server
+
+1. First [Change cgroup version] to v1 for compatibility with systemd in containers on
+   modern Linux distributions, where cgroups v2 are enabled by default:
+
+    ```bash
+    source /etc/default/grub
+    sudo sed -i '/GRUB_CMDLINE_LINUX/d' /etc/default/grub
+    echo GRUB_CMDLINE_LINUX=$(printf '\"%s systemd.unified_cgroup_hierarchy=0\"\n' "${GRUB_CMDLINE_LINUX}") \
+      | sudo tee -a /etc/default/grub
+    sudo update-grub
+    sudo reboot
+    ```
+
+2. Ensure cgroups v2 is disabled
+
+    ```bash
+    if [ ! -f /sys/fs/cgroup/cgroup.controllers ]; then
+        echo "cgroups v2 is disabled"
+    else
+        echo "cgroups v2 is enabled"
+    fi
+    ```
+
+3. Now, install or update essential software:
+
+    ```bash
+    sudo apt-get update && sudo apt-get install -y make openssl git jq
+    ```
+
+4. Install Docker Engine
+
+    ```bash
+    which docker || curl -fsSL https://get.docker.com | sh -
+    ```
+
+5. Create a new user with appropriate permissions:
+
+    ```bash
+    sudo useradd -s /bin/bash -m -G docker,sudo balena
+    echo 'balena ALL=(ALL) NOPASSWD: ALL' | tee >/etc/sudoers.d/balena
+    ```
+
+6. Switch user:
+
+   ```bash
+   sudo su balena
+   ```
+
+7. Clone the openBalena repository and change directory:
+
+   ```bash
+   git clone https://github.com/balena-io/open-balena.git ~/open-balena
+   cd ~/open-balena
+   ```
+
+8. Start the server on your domain name:
+
+   ```bash
+   export DNS_TLD=mydomain.com
+   make up
+   ```
+
+   Note down `SUPERUSER_EMAIL` and `SUPERUSER_PASSWORD` values to be used later.
+
+9. Tail the logs of the containers with:
+
+   ```bash
+   docker compose logs -f api
+   ```
+
+   Replace `api` with the name of any one of the services from the [composition].
+
+10. The server can be stopped with:
+
+   ```bash
+   make down
+   ```
+
+   The server can also be restarted using `make restart`.
+
+To update openBalena, run:
+
+```bash
+make update
+```
+
+### Test the openBalena server
+
+To confirm that everything is running correctly, try a simple request from the local
+machine to the server after registering its CA certificate(s) with the host:
+
+```bash
+make self-signed
+make verify
+```
+
+Note, if you've previously stopped the server with `make down`, run `make up` again first.
+
+Congratulations! The openBalena server is up and running. The next step is to setup your
+local machine to use this server, provision a device and deploy a small project.
+
+### Install self-signed certificates on the local machine.
+
+The installation of the openBalena server produces a self-signed certificate by default,
+which must be trusted by all devices communicating with it. This type of configuration is
+not recommended for production deployments, skip to [SSL Configuration](#ssl-configuration)
+instead.
+
+The root CA bundle can be found at `.balena/ca-${DNS_TLD}.pem` on the server. Follow the
+steps below for your specific local machine platform after manually copying it across.
+
+#### Linux:
+
+```bash
+sudo cp ca.pem /usr/local/share/ca-certificates/
+sudo update-ca-certificates
+sudo systemctl restart docker
+```
+
+#### macOS:
+
+```bash
+sudo security add-trusted-cert -d \
+  -r trustRoot \
+  -k /Library/Keychains/System.keychain \
+  ca.pem
+
+curl http://localhost/engine/restart \
+  -H 'Content-Type: application/json' \
+  -d '{"openContainerView": true}' \
+  --unix-socket ~/Library/Containers/com.docker.docker/Data/backend.sock
+```
+
+#### Windows:
+
+```PowerShell
+certutil -addstore -f "ROOT" ca.pem
+Stop-Service -Name Docker
+Start-Service -Name Docker
+```
+
+### SSL Configuration
+
+opeBalena server now uses automatic SSL configuration via ACME [DNS-01] challenge[^1]. Support
+for the following DNS providers is currently implemented:
+
+* Cloudflare
+* Gandi
+
+#### Cloudflare
+
+Obtain a Cloudflare API token with write access to your openBalena domain name records:
+
+```bash
+export ACME_EMAIL=acme@mydomain.com
+export CLOUDFLARE_API_TOKEN={{token}}
+```
+
+#### Gandi
+
+Obtain a Gandi API token with write access to your openBalena domain name records:
+
+```bash
+export ACME_EMAIL=acme@mydomain.com
+export GANDI_API_TOKEN={{token}}
+```
+
+#### Re-configure and test the server
+
+```bash
+make auto-pki
+make verify
+```
+
+#### Custom SSL
+
+openBalena server also supports custom/manual TLS configuration. You must supply your own
+SSL certificate, private key and a full certificate signing chain. A wildcard SSL
+certificate covering the whole domain is recommended.
+
+1. After obtaining your certificate, run the following commands on openBalena server:
+
+```bash
+export HAPROXY_CRT="{{ base64 encoded server certificate }}"
+export ROOT_CA="{{ .. intermediate certificates }}"
+export HAPROXY_KEY="{{ .. private key }}"
+```
+
+Pipe the plaintext via `.. | openssl base64 -A` to encode.
+
+2. Re-configure and test the server:
+
+```bash
+make pki-custom
+make verify
+```
+
+### Install the balena CLI on the local machine
+
+Follow the [balena CLI installation instructions] to install the balena CLI on the local
+machine.
+
+By default, the CLI targets the balenaCloud servers at `balena-cloud.com`, and
+needs to be configured to target the openBalena server instead. Add the following
+line to the CLI's configuration file, replacing `"mydomain.com"` with the domain
+name of the openBalena server:
+
+```yaml
+balenaUrl: 'mydomain.com'
+```
+
+The CLI configuration file can be found at:
+
+- On Linux or macOS: `~/.balenarc.yml`
+- On Windows: `%UserProfile%\_balenarc.yml`
+
+If the file does not already exist, just create it. Alternatively, `BALENARC_BALENA_URL`
+environment variable can be set to point to `"mydomain.com"`.
+
+Wrapping up the CLI installation, set an environment variable that points to the
+root certificate copied previously on the local machine. This step is to ensure
+the CLI can securely interact with the openBalena server when running self-signed PKI.
+This step can be skipped if the server is operating with publicly trusted PKI.
+
+| Shell              | Command                                        |
+| ------------------ | ---------------------------------------------- |
+| bash               | `export NODE_EXTRA_CA_CERTS='/path/to/ca.pem'` |
+| Windows cmd.exe    | `set NODE_EXTRA_CA_CERTS=C:\path\to\ca.pem`    |
+| Windows PowerShell | `$Env:NODE_EXTRA_CA_CERTS="C:\path\to\ca.pem"` |
+
+### Deploy an application
+
+The commands below should be run on a terminal on the local machine (where the
+balena CLI is installed). Ensure that the `NODE_EXTRA_CA_CERTS` environment
+variable is set, as discussed above.
+
+#### Login to openBalena
+
+Run `balena login`, select `Credentials` and use `SUPERUSER_EMAIL` and
+`SUPERUSER_PASSWORD` generated during `make up` step to login to the openBalena server.
+At any time, `balena whoami` command may be used to check which server the CLI is
+authenticated with.
+
+#### Create an application
+
+Create a new application with `balena fleet create myApp`. Select the application's
+default device type with the interactive prompt. The examples in this guide assume
+a Raspberry Pi 3.
+
+An application contains devices that share the same architecture (such as ARM or Intel),
+and also contains code releases that are deployed to the devices. When a device is
+provisioned, it is added to an application, but can be migrated to another application at
+any time. There is no limit to the number of applications that can be created or to the
+number of devices that can be provisioned.
+
+At any time, the server can be queried for all the applications it knows about
+with the following command:
+
+```bash
+balena fleets
+ Id App name Slug        Device type  Device count Online devices
+ ── ──────── ─────────── ──────────── ──────────── ──────────────
+ 1  myApp    admin/myapp raspberrypi3 0            0
+```
+
+#### Provision a new device
+
+Once we have an application, it’s time to start provisioning devices. To do this,
+first download a [balenaOS] image for your device. For this example we are using a
+Raspberry Pi 3.
+
+Unzip the downloaded image and use the balena CLI to configure it:
+
+```bash
+balena os configure --dev --fleet myApp ~/Downloads/raspberrypi3-5.2.8-v16.1.10.img
+```
+
+Flash the configured image to an SD card using [Etcher] or balena CLI:
+
+```bash
+sudo balena local flash ~/Downloads/raspberrypi3-5.2.8-v16.1.10.img
+```
+
+Insert the SD card into the device and power it on. The device will register with the
+openBalena server and after about two minutes will be inspectable:
+
+```bash
+balena devices
+ID UUID    DEVICE NAME DEVICE TYPE  FLEET       STATUS IS ONLINE SUPERVISOR VERSION OS VERSION
+1  560dcc2 quiet-rock  raspberrypi3 admin/myapp Idle   true      16.1.10            balenaOS 5.2.8
+
+balena device 560dcc2
+== WANDERING RAIN
+ID:                    1
+DEVICE TYPE:           raspberrypi3
+STATUS:                idle
+IS ONLINE:             true
+IP ADDRESS:            192.168.1.42
+MAC ADDRESS:           B8:27:DE:AD:BE:EF
+FLEET:                 admin/myapp
+LAST SEEN:             1977-08-20T14:29:00.042Z
+UUID:                  560dcc24b221c8a264d5bd981284801f
+COMMIT:                N/a
+SUPERVISOR VERSION:    16.1.10
+IS WEB ACCESSIBLE:     false
+OS VERSION:            balenaOS 5.2.8
+DASHBOARD URL:         https://dashboard.mydomain.com/devices/560dcc24b221c8a264d5bd981284801f/summary
+CPU USAGE PERCENT:     2
+CPU TEMP C:            39
+CPU ID:                00000000335956af
+MEMORY USAGE MB:       140
+MEMORY TOTAL MB:       971
+MEMORY USAGE PERCENT:  14
+STORAGE BLOCK DEVICE:  /dev/mmcblk0p6
+STORAGE USAGE MB:      76
+STORAGE TOTAL MB:      14121
+STORAGE USAGE PERCENT: 1
+```
+
+Note, even though the dashboard URL is populated, there is no dashboard service in
+openBalena.
+
+It's time to deploy code to the device.
+
+#### Deploy a project
+
+Application release images are built on the local machine using the balena CLI. Ensure the
+root certificate has been correctly installed on the local machine, as discussed above.
+
+Let's create a trivial project that logs "Idling...". On an empty directory, create a new
+file named `Dockerfile.template` with the following contents:
+
+```dockerfile
+FROM balenalib/%%BALENA_MACHINE_NAME%%-alpine
+
+CMD [ "balena-idle" ]
+```
+
+Then build and deploy the project with:
+
+```bash
+balena deploy --noparent-check myApp
+```
+
+The project will have been successfully built when a friendly unicorn appears in the
+terminal:
+
+```bash
+[Info]    No "docker-compose.yml" file found at "~/open-balena/balena-idle"
+[Info]    Creating default composition with source: "~/open-balena/balena-idle"
+[Info]    Everything is up to date (use --build to force a rebuild)
+[Info]    Creating release...
+[Info]    Pushing images to registry...
+[Info]    Saving release...
+[Success] Deploy succeeded!
+[Success] Release: 50be7bdb0ea6819c91a5dd7bcd7635ad
+
+                            \
+                             \
+                              \\
+                               \\
+                                >\/7
+                            _.-(6'  \
+                           (=___._/` \
+                                )  \ |
+                               /   / |
+                              /    > /
+                             j    < _\
+                         _.-' :      ``.
+                         \ r=._\        `.
+                        <`\\_  \         .`-.
+                         \ r-7  `-. ._  ' .  `\
+                          \`,      `-.`7  7)   )
+                           \/         \|  \'  / `-._
+                                      ||    .'
+                                       \\  (
+                                        >\  >
+                                    ,.-' >.'
+                                   <.'_.''
+                                     <'
+```
+
+This command packages up the local directory, creates a new Docker image from it and
+pushes it to the openBalena server. In turn, the server will deploy it to all provisioned
+devices and within a couple of minutes, they will all run the new release. Logs can be
+viewed with:
+
+```bash
+balena logs --tail 560dcc2
+[Logs]    [2024-05-02T15:59:31.383Z] Supervisor starting
+[Logs]    [2024-05-02T15:59:37.552Z] Applying configuration change {"SUPERVISOR_VPN_CONTROL":"true"}
+[Logs]    [2024-05-02T15:59:37.599Z] Applied configuration change {"SUPERVISOR_VPN_CONTROL":"true"}
+[Logs]    [2024-05-02T15:59:40.331Z] Creating network 'default'
+[Logs]    [2024-05-02T16:11:15.331Z] Supervisor starting
+[Logs]    [2024-05-02T16:44:08.199Z] Creating volume 'resin-data'
+[Logs]    [2024-05-02T16:44:08.572Z] Downloading image 'registry2.mydomain.com/v2/…
+…
+[Logs]    [2024-05-02T16:44:37.200Z] [main] Idling...
+[Logs]    [2024-05-02T16:44:37.200Z] [main] Idling...
+```
+
+Enjoy Balenafying All the Things!
+
+## Next steps
+
+- Try out [local mode], which allows you to build and sync code to your device locally for
+  rapid development.
+- Develop an application with [multiple containers] to provide a more modular approach to
+  application management.
+- Manage your device fleet with the use of [configuration] and [environment] variables.
+- Explore our [example projects] to give you an idea of more things you can do with
+  balena.
+- If you find yourself stuck or confused, help is just [a click away].
+- Pin selected devices to selected code releases using [sample scripts].
+- To change the superuser password after setting the credentials, follow this [forum post]
+
+
+[^1]: If DNS validation is not an option, [acme.sh] or [certbot] can be used to manually
+issue a certificate, which can then be set using the [custom SSL](#custom-ssl) workflow.
+
+
+[local mode]: https://www.balena.io/docs/learn/develop/local-mode
+[multiple containers]: https://www.balena.io/docs/learn/develop/multicontainer
+[configuration]: https://www.balena.io/docs/learn/manage/configuration
+[environment]: https://www.balena.io/docs/learn/manage/serv-vars
+[example projects]: https://balena.io/blog/tags/etcher-featured
+[a click away]: https://www.balena.io/support
+[sample scripts]: https://github.com/balena-io-examples/staged-releases
+[forum post]: https://forums.balena.io/t/upate-superuser-password/4738/6
+[balena CLI installation instructions]: https://github.com/balena-io/balena-cli/blob/master/INSTALL.md
+[Etcher]: https://balena.io/etcher
+[balenaOS]: https://balena.io/os/#download
+[balenaEngine]: https://www.balena.io/engine
+[Docker]: https://docs.docker.com/get-docker
+[Docker Engine]: https://docs.docker.com/engine/install
+[Change cgroup version]: https://docs.docker.com/config/containers/runmetrics/#changing-cgroup-version
+[composition]: https://github.com/balena-io/open-balena/blob/master/docker-compose.yml
+[DNS-01]: https://letsencrypt.org/docs/challenge-types/#dns-01-challenge
+[acme.sh]: https://github.com/acmesh-official/acme.sh
+[certbot]: https://certbot.eff.org/

repo.yml

@@ -1,5 +1,5 @@
-type: "generic"
-reviewers: 1
+---
+type: generic
 upstream:
   - repo: open-balena-api
     url: https://github.com/balena-io/open-balena-api
@@ -11,5 +11,3 @@ upstream:
     url: https://github.com/balena-io/open-balena-db
   - repo: open-balena-s3
     url: https://github.com/balena-io/open-balena-s3
-  - repo: balena-mdns-publisher
-    url: https://github.com/balena-io/balena-mdns-publisher

scripts/_keyid.js

@@ -1,79 +0,0 @@
-'use strict';
-
-var crypto = require('crypto');
-var fs = require('fs');
-
-var base32 = (function() {
-  // Extracted from https://github.com/chrisumbel/thirty-two
-  // to avoid having to install packages for this script.
-  var charTable = "ABCDEFGHIJKLMNOPQRSTUVWXYZ234567";
-  var byteTable = [
-    0xff, 0xff, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f,
-    0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-    0xff, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06,
-    0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e,
-    0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16,
-    0x17, 0x18, 0x19, 0xff, 0xff, 0xff, 0xff, 0xff,
-    0xff, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06,
-    0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e,
-    0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16,
-    0x17, 0x18, 0x19, 0xff, 0xff, 0xff, 0xff, 0xff
-  ];
-
-  function quintetCount(buff) {
-    var quintets = Math.floor(buff.length / 5);
-    return buff.length % 5 == 0 ? quintets: quintets + 1;
-  }
-
-  return function(plain) {
-    if (!Buffer.isBuffer(plain)) {
-      plain = new Buffer(plain);
-    }
-    var i = 0;
-    var j = 0;
-    var shiftIndex = 0;
-    var digit = 0;
-    var encoded = new Buffer(quintetCount(plain) * 8);
-
-    /* byte by byte isn't as pretty as quintet by quintet but tests a bit
-      faster. will have to revisit. */
-    while(i < plain.length) {
-      var current = plain[i];
-
-      if(shiftIndex > 3) {
-        digit = current & (0xff >> shiftIndex);
-        shiftIndex = (shiftIndex + 5) % 8;
-        digit = (digit << shiftIndex) | ((i + 1 < plain.length) ?
-          plain[i + 1] : 0) >> (8 - shiftIndex);
-        i++;
-      } else {
-        digit = (current >> (8 - (shiftIndex + 5))) & 0x1f;
-        shiftIndex = (shiftIndex + 5) % 8;
-        if(shiftIndex == 0) i++;
-      }
-
-      encoded[j] = charTable.charCodeAt(digit);
-      j++;
-    }
-
-    for (i = j; i < encoded.length; i++) {
-      encoded[i] = 0x3d; //'='.charCodeAt(0)
-    }
-    return encoded;
-  }
-})();
-
-function joseKeyId(der) {
-  var hasher = crypto.createHash('sha256');
-  hasher.update(der);
-  var b32 = base32(hasher.digest().slice(0, 30)).toString('ascii');
-  var chunks = [];
-  for (var i = 0; i < b32.length; i += 4) {
-    chunks.push(b32.substr(i, 4));
-  }
-  return chunks.join(':');
-}
-
-var derFilePath = process.argv[2];
-var der = fs.readFileSync(derFilePath);
-process.stdout.write(joseKeyId(der));

scripts/_realpath

@@ -1,35 +0,0 @@
-#!/bin/bash -e
-
-echo_error() {
-    local RED=`tput setaf 1`
-    local RESET=`tput sgr0`
-    echo "${RED}ERROR: ${1}${RESET}"
-}
-
-REALPATH=
-REALPATHS=(
-    'realpath'
-    'grealpath'
-    'greadlink -f'
-)
-for cmd in "${REALPATHS[@]}"; do
-if command -v "${cmd%% *}" &>/dev/null; then
-    REALPATH="${cmd}"
-    break
-fi
-done
-
-if [ -z "${REALPATH}" ]; then
-    echo_error 'Unable to find suitable command for realpath.'
-    if [ $(uname) == 'Darwin' ]; then
-        echo 'GNU coreutils are required to build openBalena on macOS. To install with brew, run'
-        echo ''
-        echo '    brew install coreutils'
-        echo ''
-    fi
-    exit 1
-fi
-
-realpath() {
-  echo $(command ${REALPATH} "$@")
-}

scripts/compose

@@ -1,47 +0,0 @@
-#!/bin/bash -e
-
-source "${BASH_SOURCE%/*}/_realpath"
-
-CMD="$(realpath "$0")"
-DIR="$(dirname "${CMD}")"
-BASE_DIR="$(dirname "${DIR}")"
-CONFIG_DIR="${BASE_DIR}/config"
-
-echo_bold() {
-  printf "\\033[1m%s\\033[0m\\n" "$@"
-}
-
-echo_bold_stderr() {
-  printf "\\033[1m%s\\033[0m\\n" "$@" 1>&2
-}
-
-VERSIONS_FILE="${BASE_DIR}/compose/versions"
-if [ ! -f "$VERSIONS_FILE" ]; then
-  echo_bold "No service versions defined in ${VERSIONS_FILE}"
-  exit 1
-fi
-
-ENV_FILE="${CONFIG_DIR}/activate"
-if [ ! -f "$ENV_FILE" ]; then
-  echo_bold 'No configuration found; please create one first with: ./scripts/quickstart'
-  echo_bold 'See README.md for help.'
-  exit 1
-fi
-
-source "${ENV_FILE}"
-
-# only include the MDNS publisher IF the domain is valid...
-if [ ${OPENBALENA_HOST_NAME: -6} == ".local" ]; then
-  INCLUDE_MDNS="-f ${BASE_DIR}/compose/mdns.yml"
-fi
-
-# show a warning to update your balena CLI tool...
-echo_bold_stderr "IMPORTANT: Please update your Balena CLI installation to version v12.38.5"
-
-# shellcheck source=/dev/null
-source "${VERSIONS_FILE}"; docker-compose \
-  --project-name 'openbalena' \
-  -f "${BASE_DIR}/compose/services.yml" \
-  ${INCLUDE_MDNS} \
-  -f "${CONFIG_DIR}/docker-compose.yml" \
-  "$@"

scripts/create-superuser

@@ -1,36 +0,0 @@
-#!/bin/bash -e
-
-usage() {
-  echo "usage: $0 EMAIL PASSWORD"
-  echo
-  echo 'Create the superuser account with the given email and password.'
-  echo
-  echo 'The instance must already be running in the background. You can '
-  echo 'start it with: ./scripts/compose up -d'
-}
-
-if [[ -z "$1" || -z "$2" ]]; then
-  usage
-  exit 1
-fi
-
-echo_bold() {
-  printf "\\033[1m%s\\033[0m\\n" "${@}"
-}
-
-source "${BASH_SOURCE%/*}/_realpath"
-
-CMD="$(realpath "$0")"
-DIR="$(dirname "${CMD}")"
-FIG="${DIR}/compose"
-
-EMAIL="$1"
-PASSWORD="$2"
-
-"${FIG}" exec api /bin/bash -c \
-  'export $(grep -v "^#" config/env | xargs -d "\n"); node index.js create-superuser root '${EMAIL}' '${PASSWORD}'' \
-  >/dev/null \
-  || (echo 'Failed to create superuser; please ensure the instance is running and that no superuser has been created before.' && exit 1)
-
-echo_bold "==> Success! Superuser created with email: ${EMAIL}"
-echo "  - You may now login with: balena login --credentials --email ${EMAIL}"

scripts/gen-root-ca

@@ -1,35 +0,0 @@
-#!/bin/bash -e
-
-usage() {
-  echo "usage: $0 COMMON_NAME [OUT]"
-  echo
-  echo "  COMMON_NAME   the domain name the certificate is valid for, eg. example.com"
-  echo "  OUT           path to output directory generated files will be placed in"
-  echo
-}
-
-if [ -z "$1" ]; then
-  usage
-  exit 1
-fi
-
-CMD="$(realpath "$0")"
-DIR="$(dirname "${CMD}")"
-
-CN="$1"
-OUT="$(realpath "${2:-.}")"
-
-# shellcheck source=scripts/ssl-common.sh
-source "${DIR}/ssl-common.sh"
-
-ROOT_CA="${ROOT_PKI}/ca.crt"
-
-if [ ! -f $ROOT_CA ]; then
-  # Create a secret key and CA file for the self-signed CA
-  "$easyrsa_bin" --pki-dir="${ROOT_PKI}" init-pki 2>/dev/null
-  "$easyrsa_bin" --pki-dir="${ROOT_PKI}" --days="${CA_EXPIRY_DAYS}" --req-cn="ca.${CN}" build-ca nopass 2>/dev/null
-
-  # update indexes and generate CRLs
-  "$easyrsa_bin" --pki-dir="${ROOT_PKI}" update-db 2>/dev/null
-  "$easyrsa_bin" --pki-dir="${ROOT_PKI}" gen-crl 2>/dev/null
-fi

scripts/gen-root-cert

@@ -1,36 +0,0 @@
-#!/bin/bash -e
-
-usage() {
-  echo "usage: $0 COMMON_NAME [OUT]"
-  echo
-  echo "  COMMON_NAME   the domain name the certificate is valid for, eg. example.com"
-  echo "  OUT           path to output directory generated files will be placed in"
-  echo
-}
-
-if [ -z "$1" ]; then
-  usage
-  exit 1
-fi
-
-CMD="$(realpath "$0")"
-DIR="$(dirname "${CMD}")"
-
-CN="$1"
-OUT="$(realpath "${2:-.}")"
-
-# shellcheck source=scripts/ssl-common.sh
-source "${DIR}/ssl-common.sh"
-
-ROOT_CRT="${ROOT_PKI}"'/issued/*.'"${CN}"'.crt'
-ROOT_KEY="${ROOT_PKI}"'/private/*.'"${CN}"'.key'
-
-if [ ! -f $ROOT_CRT ] || [ ! -f $ROOT_KEY ]; then
-  rm -f $ROOT_CRT $ROOT_KEY
-  # generate default CSR and sign (root + wildcard)
-  "$easyrsa_bin" --pki-dir="${ROOT_PKI}" --days="${CRT_EXPIRY_DAYS}" --subject-alt-name="DNS:*.${CN}" build-server-full "*.${CN}" nopass 2>/dev/null
-
-  # update indexes and generate CRLs
-  "$easyrsa_bin" --pki-dir="${ROOT_PKI}" update-db 2>/dev/null
-  "$easyrsa_bin" --pki-dir="${ROOT_PKI}" gen-crl 2>/dev/null
-fi;

scripts/gen-token-auth-cert

@@ -1,53 +0,0 @@
-#!/bin/bash -e
-
-usage() {
-  echo "usage: $0 COMMON_NAME [OUT]"
-  echo
-  echo "  COMMON_NAME   the domain name the certificate is valid for, eg. example.com"
-  echo "  OUT           path to output directory generated files will be placed in"
-  echo
-}
-
-if [ -z "$1" ]; then
-  usage
-  exit 1
-fi
-
-CMD="$(realpath "$0")"
-DIR="$(dirname "${CMD}")"
-
-CN="$1"
-OUT="$(realpath "${2:-.}")"
-
-# shellcheck source=scripts/ssl-common.sh
-source "${DIR}/ssl-common.sh"
-
-CERT_DIR="${OUT}/api"
-CERT_FILE="${CERT_DIR}/api.${CN}"
-
-keyid() {
-  # NodeJS is installed as `nodejs` in some distros, `node` in others.
-  node_bin="$(command -v nodejs 2>/dev/null || command -v node 2>/dev/null || true)"
-  if [ -z "$node_bin" ]; then
-    echo >&2 'NodeJS is required but not installed. Aborting.'
-    exit 1
-  fi
-  # Recent Node versions complain about `new Buffer()` being deprecated
-  # but the alternative is not available to older versions. Silence the
-  # warning but use the deprecated form to allow greater compatibility.
-  "$node_bin" --no-deprecation "${DIR}/_keyid.js" "$1"
-}
-
-JWT_CRT="${CERT_FILE}.crt"
-JWT_KEY="${CERT_FILE}.pem"
-JWT_KID="${CERT_FILE}.kid"
-
-if [ ! -f $JWT_CRT ] || [ ! -f $JWT_KEY ] || [ ! -f $JWT_KID ]; then
-  rm -f $JWT_CRT $JWT_KEY $JWT_KID
-  mkdir -p "${CERT_DIR}"
-  openssl ecparam -name prime256v1 -genkey -noout -out "${JWT_KEY}" 2>/dev/null
-  openssl req -x509 -new -nodes -days "${CRT_EXPIRY_DAYS}" -key "${JWT_KEY}" -subj "/CN=api.${CN}" -out "${JWT_CRT}" 2>/dev/null
-  openssl ec -in "${JWT_KEY}" -pubout -outform DER -out "${CERT_FILE}.der" 2>/dev/null
-  keyid "${CERT_FILE}.der" >"${JWT_KID}"
-  rm "${CERT_FILE}.der"
-fi

scripts/gen-vpn-certs

@@ -1,48 +0,0 @@
-#!/bin/bash -e
-
-usage() {
-  echo "usage: $0 COMMON_NAME [OUT]"
-  echo
-  echo "  COMMON_NAME   the domain name the certificate is valid for, eg. example.com"
-  echo "  OUT           path to output directory generated files will be placed in"
-  echo
-}
-
-if [ -z "$1" ]; then
-  usage
-  exit 1
-fi
-
-CMD="$(realpath "$0")"
-DIR="$(dirname "${CMD}")"
-
-CN="$1"
-OUT="$(realpath "${2:-.}")"
-
-# shellcheck source=scripts/ssl-common.sh
-source "${DIR}/ssl-common.sh"
-
-VPN_PKI="$(realpath "${OUT}/vpn")"
-VPN_CA="${VPN_PKI}/ca.crt"
-VPN_CRT="${VPN_PKI}/issued/vpn.${CN}.crt"
-VPN_KEY="${VPN_PKI}/private/vpn.${CN}.key"
-VPN_DH="${VPN_PKI}/dh.pem"
-
-if [ ! -f $VPN_CA ] || [ ! -f $VPN_CRT ] || [ ! -f $VPN_KEY ] || [ ! -f $VPN_DH ]; then
-
-  rm -f $VPN_CA $VPN_CRT $VPN_DH $VPN_KEY
-
-  # generate VPN CA
-  "$easyrsa_bin" --pki-dir="${VPN_PKI}" init-pki &>/dev/null
-  "$easyrsa_bin" --pki-dir="${VPN_PKI}" --days="${CA_EXPIRY_DAYS}" --req-cn="vpn-ca.${CN}" build-ca nopass 2>/dev/null
-
-  # generate and sign vpn server certificate
-  "$easyrsa_bin" --pki-dir="${VPN_PKI}" --days="${CRT_EXPIRY_DAYS}" build-server-full "vpn.${CN}" nopass 2>/dev/null
-
-  # generate vpn dhparams (keysize of 2048 will do, 4096 can wind up taking hours to generate)
-  "$easyrsa_bin" --pki-dir="${VPN_PKI}" --keysize=2048 gen-dh 2>/dev/null
-
-  # update indexes and generate CRLs
-  "$easyrsa_bin" --pki-dir="${VPN_PKI}" update-db 2>/dev/null
-  "$easyrsa_bin" --pki-dir="${VPN_PKI}" gen-crl 2>/dev/null
-fi

scripts/logger.sh

@@ -1,62 +0,0 @@
-#!/bin/sh
-
-BLACK=`tput setaf 0`
-RED=`tput setaf 1`
-GREEN=`tput setaf 2`
-YELLOW=`tput setaf 3`
-BLUE=`tput setaf 4`
-MAGENTA=`tput setaf 5`
-CYAN=`tput setaf 6`
-WHITE=`tput setaf 7`
-
-BOLD=`tput bold`
-RESET=`tput sgr0`
-
-log_raw () {
-    local COLOR="${WHITE}"
-    local LEVEL="${1}"
-    local MESSAGE="${2}"
-    case "${LEVEL}" in
-        info)
-            COLOR="${BLUE}"
-            ;;
-        warn)
-            COLOR="${YELLOW}"
-            ;;
-        fatal)
-            COLOR="${RED}"
-            ;;
-        *)
-            LEVEL="debug"
-            ;;
-    esac
-    LEVEL="${LEVEL}     "
-    echo "[$(date +%T)] ${COLOR}$(echo "${LEVEL:0:5}" | tr '[:lower:]' '[:upper:]')${RESET} ${MESSAGE}";
-}
-
-log () {
-    log_raw "debug" "${1}"
-}
-
-info () {
-    log_raw "info" "${1}";
-}
-
-warn () {
-    log_raw "warn" "${1}";
-}
-
-die () {
-    log_raw "fatal" "${1}";
-    exit 1;
-}
-
-die_unless_forced () {
-    if [ ! -z "$1" ]; then
-        log_raw "warn" "$2";
-        return;
-    fi
-    
-    log_raw "fatal" "$2";
-    die "Use -f to forcibly upgrade.";
-}

scripts/make-env

@@ -1,77 +0,0 @@
-#!/bin/bash -e
-
-usage() {
-  echo "usage: $0"
-  echo
-  echo "Required Variables:"
-  echo
-  echo "  DOMAIN"
-  echo "  ROOT_CA    Path to root CA certificate"
-  echo "  ROOT_CRT   Path to root/wildcard certificate"
-  echo "  ROOT_KEY   Path to root/wildcard private key"
-  echo "  JWT_CRT    Path to Token Auth certificate"
-  echo "  JWT_KEY    Path to Token Auth private key"
-  echo "  JWT_KID    Path to KeyID for the Token Auth certificate"
-  echo "  VPN_CA     Path to the VPN CA certificate"
-  echo "  VPN_CRT    Path to the VPN server certificate"
-  echo "  VPN_KEY    Path to the VPN server private key"
-  echo "  VPN_DH     Path to the VPN server Diffie Hellman parameters"
-  echo "  SUPERUSER_EMAIL     Email address of the superuser"
-  echo "  SUPERUSER_PASSWORD  Password of the superuser"
-  echo
-}
-
-for var in DOMAIN ROOT_CA ROOT_CRT ROOT_KEY JWT_CRT JWT_KEY JWT_KID VPN_CA VPN_CRT VPN_KEY VPN_DH SUPERUSER_EMAIL SUPERUSER_PASSWORD; do
-  if [ -z "${!var-}" ]; then
-    usage
-    exit 1
-  fi
-done
-
-randstr() {
-  LC_CTYPE=C tr -dc A-Za-z0-9 < /dev/urandom | fold -w "${1:-32}" | head -n 1
-}
-
-b64encode() {
-    echo "$@" | base64 --wrap=0 2>/dev/null || echo "$@" | base64 --break=0 2>/dev/null
-}
-
-b64file() {
-  b64encode "$(cat "$@")"
-}
-
-# buckets to create in the S3 service...
-REGISTRY2_S3_BUCKET="registry-data"
-
-cat <<STR
-export OPENBALENA_PRODUCTION_MODE=false
-export OPENBALENA_COOKIE_SESSION_SECRET=$(randstr 32)
-export OPENBALENA_HOST_NAME=$DOMAIN
-export OPENBALENA_JWT_SECRET=$(randstr 32)
-export OPENBALENA_REGISTRY2_S3_BUCKET=${REGISTRY2_S3_BUCKET}
-export OPENBALENA_RESINOS_REGISTRY_CODE=$(randstr 32)
-export OPENBALENA_ROOT_CA=$(b64file "${ROOT_CA}")
-export OPENBALENA_ROOT_CRT=$(b64file "${ROOT_CRT}")
-export OPENBALENA_ROOT_KEY=$(b64file "${ROOT_KEY}")
-export OPENBALENA_TOKEN_AUTH_BUILDER_TOKEN=$(randstr 64)
-export OPENBALENA_TOKEN_AUTH_PUB=$(b64file "$JWT_CRT")
-export OPENBALENA_TOKEN_AUTH_KEY=$(b64file "$JWT_KEY")
-export OPENBALENA_TOKEN_AUTH_KID=$(b64file "$JWT_KID")
-export OPENBALENA_VPN_CA=$(b64file "$VPN_CA")
-export OPENBALENA_VPN_CA_CHAIN=$(b64file "$VPN_CA")
-export OPENBALENA_VPN_SERVER_CRT=$(b64file "$VPN_CRT")
-export OPENBALENA_VPN_SERVER_KEY=$(b64file "$VPN_KEY")
-export OPENBALENA_VPN_SERVER_DH=$(b64file "$VPN_DH")
-export OPENBALENA_VPN_SERVICE_API_KEY=$(randstr 32)
-export OPENBALENA_API_VPN_SERVICE_API_KEY=$(randstr 32)
-export OPENBALENA_REGISTRY_SECRET_KEY=$(randstr 32)
-export OPENBALENA_S3_ACCESS_KEY=$(randstr 32)
-export OPENBALENA_S3_BUCKETS="${REGISTRY2_S3_BUCKET}"
-export OPENBALENA_S3_ENDPOINT="https://s3.${DOMAIN}"
-export OPENBALENA_S3_REGION=us-east-1
-export OPENBALENA_S3_SECRET_KEY=$(randstr 32)
-export OPENBALENA_SSH_AUTHORIZED_KEYS=
-export OPENBALENA_SUPERUSER_EMAIL=$SUPERUSER_EMAIL
-export OPENBALENA_SUPERUSER_PASSWORD=$(printf "%q" "${SUPERUSER_PASSWORD}")
-export OPENBALENA_ACME_CERT_ENABLED=${ACME_CERT_ENABLED:-false}
-STR

scripts/migrate-registry-storage

@@ -1,29 +0,0 @@
-#!/bin/sh
-
-migrate_data_to_s3 () {
-  BUCKET="${1:-registry-data}"
-
-  if [ -z "${BUCKET}" ]; then return 1; fi
-
-  if [ -n "${DOCKER_HOST}" ]; then
-      log "Using docker host: ${DOCKER_HOST}"
-      export DOCKER_HOST="${DOCKER_HOST}"
-  fi
-
-  REGISTRY_CONTAINER="$(docker ps | grep registry_ | awk '{print $1}')"
-  S3_CONTAINER="$(docker ps | grep s3_ | awk '{print $1}')"
-
-  if [ -z "${REGISTRY_CONTAINER}" ] || [ -z "${S3_CONTAINER}" ]; then return 2; fi
-
-  REGISTRY_VOLUME="$(docker inspect "${REGISTRY_CONTAINER}" | jq -r '.[].Mounts | map(select(.Destination=="/data")) | .[0].Source')"
-  S3_VOLUME=$(docker inspect "${S3_CONTAINER}" | jq -r '.[].Mounts | map(select(.Destination=="/export")) | .[0].Source')
-
-  if [ -z "${REGISTRY_VOLUME}" ] || [ -z "${S3_VOLUME}" ]; then return 3; fi
-
-  # run the S3 container image, and copy the data partition into S3...
-  docker run -it --rm \
-      -v "${REGISTRY_VOLUME}:/data" \
-      -v "${S3_VOLUME}:/s3" \
-      --name "migrate-registry" alpine \
-      sh -c "mkdir -p /s3/${BUCKET}/data && cp -r /data/docker /s3/${BUCKET}/data/"
-}

scripts/patch-hosts

@@ -1,30 +0,0 @@
-#!/bin/bash -e
-
-usage() {
-  echo "usage: $0 DOMAIN"
-  echo
-  echo "  DOMAIN  the domain name to add host entries for, eg. example.com"
-  echo
-}
-
-if [ -z "$1" ]; then
-  usage
-  exit 1
-fi
-
-SERVICES="api registry vpn db s3 redis"
-DOMAIN="$1"
-
-# We need sudo to write to /etc/hosts, so first write to a temp file and then
-# append all entries to hosts file.
-tmp="$(mktemp --tmpdir openbalena.XXXX)"
-for service in $SERVICES; do
-  name="${service}.${DOMAIN}"
-  if ! grep "\\s$name" /etc/hosts >/dev/null 2>&1 ; then
-    echo "adding $name"
-    echo "127.0.0.1 $name" >>"${tmp}"
-  fi
-done
-# shellcheck disable=SC2024
-sudo tee -a /etc/hosts >/dev/null <"${tmp}"
-rm -f "${tmp}"

scripts/quickstart

@@ -1,155 +0,0 @@
-#!/bin/bash -e
-
-BLACK=`tput setaf 0`
-RED=`tput setaf 1`
-GREEN=`tput setaf 2`
-YELLOW=`tput setaf 3`
-BLUE=`tput setaf 4`
-MAGENTA=`tput setaf 5`
-CYAN=`tput setaf 6`
-WHITE=`tput setaf 7`
-
-BOLD=`tput bold`
-RESET=`tput sgr0`
-
-# for macos machines, we need proper OpenSSL...
-OPENSSL_VERSION=$(openssl version -v)
-if [[ "${OPENSSL_VERSION}" =~ ^LibreSSL.*$ ]]; then
-  echo -e "${RED}ERROR: You may not have a compatible OpenSSL version (${OPENSSL_VERSION}). Please install OpenSSL version 1.0.2q or above.${RESET}"
-  if [ $(uname) == 'Darwin' ]; then
-        echo 'OpenSSL is required to build openBalena on macOS. To install with brew, run'
-        echo ''
-        echo '    brew install openssl'
-        echo ''
-    fi
-  exit 1
-fi
-
-source "${BASH_SOURCE%/*}/_realpath"
-
-domainResolves() {
-  getent hosts "$1" > /dev/null 2>&1
-}
-
-CMD="$(realpath "$0")"
-DIR="$(dirname "${CMD}")"
-BASE_DIR="$(dirname "${DIR}")"
-CONFIG_DIR="${BASE_DIR}/config"
-CERTS_DIR="${CONFIG_DIR}/certs"
-
-DOMAIN=openbalena.local
-
-usage() {
-  echo "usage: $0 [-c] [-h] [-p] [-d DOMAIN] -U EMAIL -P PASSWORD"
-  echo
-  echo "  -c           enable the ACME certificate service in staging or production mode."
-  echo "  -p           patch hosts - patch the host /etc/hosts file"
-  echo "  -d DOMAIN    the domain name this deployment will run as, eg. example.com. Default is 'openbalena.local'"
-  echo "  -U EMAIL     the email address of the superuser account, used to login to your install from the Balena CLI"
-  echo "  -P PASSWORD  the password to use for the superuser account."
-  echo
-}
-
-show_help=false
-patch_hosts=false
-while getopts ":chpxd:U:P:" opt; do
-  case "${opt}" in
-    h) show_help=true;;
-    p) patch_hosts=true;;
-    x) set -x;;
-    d) DOMAIN="${OPTARG}";;
-    U) SUPERUSER_EMAIL="${OPTARG}";;
-    P) SUPERUSER_PASSWORD="${OPTARG}";;
-    c) ACME_CERT_ENABLED="true";;
-    *)
-      echo "Invalid argument: -${OPTARG}"
-      usage
-      exit 1
-      ;;
-  esac
-done
-shift $((OPTIND-1))
-
-if [ -z "${SUPERUSER_EMAIL}" ] || [ -z "${SUPERUSER_PASSWORD}" ]; then
-  usage
-  exit 1
-fi
-
-if [ "$show_help" = "true" ]; then
-  usage
-  exit 1
-fi
-
-if [ ! -z "$ACME_CERT_ENABLED" ]; then
-  echo "${BLUE}[INFO]${RESET} ACME Certificate request is ${BOLD}ENABLED${RESET}."
-  
-  if ! domainResolves "api.${DOMAIN}"; then
-    echo "${YELLOW}[WARN]${RESET} Unable to resolve \"api.${DOMAIN}\"!"
-    echo "${YELLOW}[WARN]${RESET} This might mean that you cannot use an ACME issued certificate."
-  fi
-fi
-
-echo_bold() {
-  echo "${BOLD}${@}${RESET}"
-}
-
-echo_bold "==> Creating new configuration at: $CONFIG_DIR"
-mkdir -p "$CONFIG_DIR" "$CERTS_DIR"
-
-echo_bold "==> Bootstrapping easy-rsa..."
-source "${DIR}/ssl-common.sh"
-
-echo_bold "==> Generating root CA cert..."
-# shellcheck source=scripts/gen-root-ca
-source "${DIR}/gen-root-ca" "${DOMAIN}" "${CERTS_DIR}"
-
-echo_bold "==> Generating root cert chain for haproxy..."
-# shellcheck source=scripts/gen-root-cert
-source "${DIR}/gen-root-cert" "${DOMAIN}" "${CERTS_DIR}"
-
-echo_bold "==> Generating token auth cert..."
-# shellcheck source=scripts/gen-token-auth-cert
-source "${DIR}/gen-token-auth-cert" "${DOMAIN}" "${CERTS_DIR}"
-
-echo_bold "==> Generating VPN CA, cert and dhparam (this may take a while)..."
-# shellcheck source=scripts/gen-vpn-certs
-source "${DIR}/gen-vpn-certs" "${DOMAIN}" "${CERTS_DIR}"
-
-echo_bold "==> Setting up environment..."
-# shellcheck source=scripts/make-env
-cat >"${CONFIG_DIR}/activate" <(source "${DIR}/make-env")
-
-echo_bold "==> Adding default compose file..."
-cp "${BASE_DIR}/compose/template.yml" "${CONFIG_DIR}/docker-compose.yml"
-
-if [ "${patch_hosts}" = "true" ]; then
-  echo_bold "==> Patching /etc/hosts..."
-  # shellcheck source=scripts/patch-hosts
-  source "${DIR}/patch-hosts" "${DOMAIN}"
-fi
-
-echo_bold "==> Success!"
-echo '  - Start the instance with: ./scripts/compose up -d'
-echo '  - Stop the instance with: ./scripts/compose stop'
-echo '  - To create a single, flat, docker-compose.yml file, run:'
-echo ''
-echo '      ./scripts/compose config > docker-compose.yml'
-echo ''
-
-if [ -z "${ACME_CERT_ENABLED}" ]; then
-  echo "  - Use the following certificate with Balena CLI: ${CERTS_DIR}/root/ca.crt"
-
-  case $(uname) in
-    Darwin)
-      echo ''
-      printf '    On macOS:\n\n'
-      printf '      sudo security add-trusted-cert -d -r trustRoot -k "/Library/Keychains/System.keychain" "%s/root/ca.crt"\n' "${CERTS_DIR}"
-      echo ''
-      ;;
-    *)
-      ;;
-  esac
-
-  echo -e "    ${YELLOW}IMPORTANT:${RESET} You will need to restart your Docker daemon after trusting this certificate to allow your workstation to push images to the registry."
-  echo ''
-fi

scripts/ssl-common.sh

@@ -1,25 +0,0 @@
-#!/bin/bash -e
-# shellcheck disable=SC2034
-
-# ensure we have `easyrsa` available
-if [ -z "${easyrsa_bin-}" ] || [ ! -x "${easyrsa_bin}" ]; then
-    easyrsa_bin="$(command -v easyrsa 2>/dev/null || true)"
-    if [ -z "${easyrsa_bin}" ]; then
-        easyrsa_dir="$(mktemp -dt easyrsa.XXXXXXXX)"
-        easyrsa_url="https://github.com/OpenVPN/easy-rsa/releases/download/v3.0.5/EasyRSA-nix-3.0.5.tgz"
-        echo "  - Downloading easy-rsa..."
-        (cd "${easyrsa_dir}"; curl -sL "${easyrsa_url}" | tar xz --strip-components=1)
-        easyrsa_bin="${easyrsa_dir}/easyrsa"
-        # shellcheck disable=SC2064
-        trap "rm -rf \"${easyrsa_dir}\"" EXIT
-    fi
-    export EASYRSA_BATCH=1
-    export EASYRSA_KEY_SIZE=4096
-fi
-
-# setup ROOT_PKI path
-ROOT_PKI="$(realpath "${OUT}/root")"
-
-# global expiry settings
-CA_EXPIRY_DAYS=3650
-CRT_EXPIRY_DAYS=730

scripts/upgrade-1.x-to-2.0

@@ -1,78 +0,0 @@
-#!/bin/sh
-
-source "${BASH_SOURCE%/*}/logger.sh"
-source "${BASH_SOURCE%/*}/migrate-registry-storage"
-
-# This script takes a v1.x.x install and updates the compose stack to use S3 as your
-# registry storage.
-
-source "${BASH_SOURCE%/*}/_realpath"
-
-DIR="$(dirname $(realpath "$0"))"
-BASE_DIR="$(dirname "${DIR}")"
-CONFIG_DIR="${BASE_DIR}/config"
-CONFIG_FILE="${CONFIG_DIR}/activate"
-
-# Step 1. Make sure a config exists...
-[ -f "${CONFIG_FILE}" ] || die "Unable to find existing config!";
-
-info "Preparing to upgrade..."
-source "${CONFIG_FILE}"
-
-while getopts "f" opt; do
-  case "${opt}" in
-    f) 
-        warn "Forcing upgrade! I hope you know what you're doing..."
-        FORCE_UPGRADE=1
-        ;;
-    *)
-      echo "Invalid argument: ${OPTARG}"
-      exit 1
-      ;;
-  esac
-done
-shift $((OPTIND-1))
-
-# Step 2. Check if the S3 configuration already exists...
-upgrade_required () {
-    [ -z "${OPENBALENA_REGISTRY2_S3_BUCKET}" ] || return 1;
-    [ -z "${OPENBALENA_S3_ACCESS_KEY}" ] || return 1;
-    [ -z "${OPENBALENA_S3_ENDPOINT}" ] || return 1;
-    [ -z "${OPENBALENA_S3_REGION}" ] || return 1;
-    [ -z "${OPENBALENA_S3_SECRET_KEY}" ] || return 1;
-}
-upgrade_required || die_unless_forced "${FORCE_UPGRADE}" "Configuration may already be using S3 for Registry storage!"
-
-# Step 3. Create missing S3 configuration...
-randstr() {
-  LC_CTYPE=C tr -dc A-Za-z0-9 < /dev/urandom | fold -w "${1:-32}" | head -n 1
-}
-
-upsert_config () {
-    var="${1}"
-    value="${2}"
-
-    if [ -z "${!var}" ]; then
-        echo "export ${1}=${2}" >> "${CONFIG_FILE}"
-    else
-        sed -i '' "s~export ${1}=.*~export ${1}=${2}~" "${CONFIG_FILE}"
-    fi
-}
-
-upsert_config "OPENBALENA_REGISTRY2_S3_BUCKET" "registry-data" || warn "Failed to update config value OPENBALENA_REGISTRY2_S3_BUCKET"
-upsert_config "OPENBALENA_S3_ACCESS_KEY" "$(randstr 32)" || warn "Failed to update config value OPENBALENA_S3_ACCESS_KEY"
-upsert_config "OPENBALENA_S3_ENDPOINT" "https://s3.${OPENBALENA_HOST_NAME}" || warn "Failed to update config value OPENBALENA_S3_ENDPOINT"
-upsert_config "OPENBALENA_S3_REGION" "us-east-1" || warn "Failed to update config value OPENBALENA_S3_REGION"
-upsert_config "OPENBALENA_S3_SECRET_KEY" "$(randstr 32)" || warn "Failed to update config value OPENBALENA_S3_SECRET_KEY"
-
-# Step 4. Migrate Registry data to S3...
-info "Copying data from the Registry volume to the S3 volume..."
-migrate_data_to_s3 "registry-data"
-case $? in
-    1)  die "Invalid bucket name";;
-    2)  die "Unable to find the running Registry or S3 containers";;
-    3)  die "Unable to determine the data volumes for the Registry or S3 containers";;
-    *)  info "Registry data copied"
-        ;;
-esac
-info "Upgrade complete"

src/README

@@ -1 +0,0 @@
-This is the working folder for any specific container you might want to work on.

src/balena-tests/Dockerfile

@@ -0,0 +1,34 @@
+FROM ubuntu:22.04
+
+# renovate: datasource=github-releases depName=balena-io/balena-cli
+ARG BALENA_CLI_VERSION=v18.2.2
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+	bash \
+	ca-certificates \
+	curl \
+	jq \
+	openssl \
+	procmail \
+	qemu-utils \
+	unzip \
+	wget \
+    && rm -rf /var/lib/apt/lists/*
+
+# https://docs.docker.com/engine/install/ubuntu/#install-using-the-convenience-script
+RUN curl -fsSL https://get.docker.com | sh
+
+WORKDIR /opt
+
+RUN set -x; arch=$(uname -m | sed 's/86_64/64/g') \
+    && wget -q "https://github.com/balena-io/balena-cli/releases/download/${BALENA_CLI_VERSION}/balena-cli-${BALENA_CLI_VERSION}-linux-${arch}-standalone.zip" \
+    && unzip -q "balena-cli-${BALENA_CLI_VERSION}-linux-${arch}-standalone.zip" \
+    && rm -rf "balena-cli-${BALENA_CLI_VERSION}-linux-${arch}-standalone.zip"
+
+ENV PATH=/opt/balena-cli:${PATH}
+
+COPY functions balena.sh /usr/sbin/
+
+WORKDIR /balena
+
+CMD /usr/sbin/balena.sh

src/balena-tests/balena.sh

@@ -0,0 +1,328 @@
+#!/usr/bin/env bash
+
+# shellcheck disable=SC2154,SC2034,SC1090
+set -ae
+
+curl_opts="--retry 3 --fail"
+if [[ $VERBOSE =~ on|On|Yes|yes|true|True ]]; then
+    set -x
+    curl_opts="${curl_opts} --verbose"
+else
+    curl_opts="${curl_opts} --silent"
+fi
+
+source /usr/sbin/functions
+
+function remove_test_assets() {
+    rm -rf /balena/config.json \
+      "${GUEST_IMAGE}" \
+      "${GUEST_IMAGE%.*}.ready" \
+      "${tmpbuild}" \
+      /tmp/*.img
+}
+
+function remove_update_lock() {
+    rm -f /tmp/balena/updates.lock
+}
+
+function cleanup() {
+   shutdown_dut
+   remove_test_assets
+   remove_update_lock
+
+   # crash loop backoff
+   sleep "$(( (RANDOM % 5) + 5 ))s"
+}
+trap 'cleanup' EXIT
+
+function shutdown_dut() {
+    local balena_device_uuid
+    balena_device_uuid="$(cat </balena/config.json | jq -r .uuid)"
+
+    if [[ -n $balena_device_uuid ]]; then
+        with_backoff balena device "${balena_device_uuid}"
+        balena device shutdown -f "${balena_device_uuid}" || true
+    fi
+}
+
+function set_update_lock {
+    if [[ -n "$BALENA_SUPERVISOR_ADDRESS" ]] && [[ -n "$BALENA_SUPERVISOR_API_KEY" ]]; then
+        while [[ $(curl ${curl_opts} "${BALENA_SUPERVISOR_ADDRESS}/v1/device?apikey=${BALENA_SUPERVISOR_API_KEY}" \
+          -H "Content-Type: application/json" | jq -r '.update_pending') == 'true' ]]; do
+
+            curl ${curl_opts} "${BALENA_SUPERVISOR_ADDRESS}/v1/device?apikey=${BALENA_SUPERVISOR_API_KEY}" \
+              -H "Content-Type: application/json" | jq -r
+
+            sleep "$(( (RANDOM % 3) + 3 ))s"
+        done
+        sleep "$(( (RANDOM % 5) + 5 ))s"
+
+        # https://www.balena.io/docs/learn/deploy/release-strategy/update-locking/
+        lockfile /tmp/balena/updates.lock
+    fi
+}
+
+function update_ca_certificates() {
+    # only set CA bundle if using private certificate chain
+    if [[ -e "${CERTS}/ca-bundle.pem" ]]; then
+        if [[ "$(readlink -f "${CERTS}/${TLD}-chain.pem")" =~ \/private\/ ]]; then
+            mkdir -p /usr/local/share/ca-certificates
+            cat <"${CERTS}/ca-bundle.pem" > /usr/local/share/ca-certificates/balenaRootCA.crt
+            # shellcheck disable=SC2034
+            CURL_CA_BUNDLE=${CURL_CA_BUNDLE:-${CERTS}/ca-bundle.pem}
+            NODE_EXTRA_CA_CERTS=${NODE_EXTRA_CA_CERTS:-${CURL_CA_BUNDLE}}
+            # (TBC) refactor to use NODE_EXTRA_CA_CERTS instead of ROOT_CA
+            # https://github.com/balena-io/e2e/blob/master/conf.js#L12-L14
+            # https://github.com/balena-io/e2e/blob/master/Dockerfile#L82-L83
+            # ... or
+            # https://thomas-leister.de/en/how-to-import-ca-root-certificate/
+            # https://github.com/puppeteer/puppeteer/issues/2377
+            ROOT_CA=${ROOT_CA:-$(cat <"${NODE_EXTRA_CA_CERTS}" | openssl base64 -A)}
+        else
+            rm -f /usr/local/share/ca-certificates/balenaRootCA.crt
+            unset NODE_EXTRA_CA_CERTS CURL_CA_BUNDLE ROOT_CA
+        fi
+        update-ca-certificates
+    fi
+}
+
+function wait_for_api() {
+    while ! curl ${curl_opts} "https://api.${DNS_TLD}/ping"; do
+        sleep "$(( (RANDOM % 5) + 5 ))s"
+    done
+}
+
+function open_balena_login() {
+    while ! balena login --credentials \
+      --email "${SUPERUSER_EMAIL}" \
+      --password "${SUPERUSER_PASSWORD}"; do
+        sleep "$(( (RANDOM % 5) + 5 ))s"
+    done
+}
+
+function create_fleet() {
+    if ! balena fleet "${TEST_FLEET}"; then
+        # wait for API to load DT contracts
+        while ! balena fleet create "${TEST_FLEET}" --type "${DEVICE_TYPE}"; do
+            sleep "$(( (RANDOM % 5) + 5 ))s"
+        done
+
+        # FIXME: on openBalena 'balena devices supported' always returns empty list
+        balena devices supported
+    fi
+}
+
+function download_os_image() {
+    if ! [[ -s "$GUEST_IMAGE" ]]; then
+        with_backoff wget -qO /tmp/balena.zip \
+          "${BALENA_API_URL}/download?deviceType=${DEVICE_TYPE}&version=${OS_VERSION:1}&fileType=.zip"
+
+        unzip -oq /tmp/balena.zip -d /tmp
+
+        cat <"$(find /tmp/ -type f -name '*.img' | head -n 1)" >"${GUEST_IMAGE}"
+
+        rm /tmp/balena.zip
+    fi
+}
+
+function configure_virtual_device() {
+    while ! [[ -s "$GUEST_IMAGE" ]]; do sleep "$(( (RANDOM % 5) + 5 ))s"; done
+
+    if ! [[ -s /balena/config.json ]]; then
+        balena_device_uuid="$(openssl rand -hex 16)"
+
+        with_backoff balena device register "${TEST_FLEET}" \
+          --uuid "${balena_device_uuid}"
+
+        with_backoff balena config generate \
+          --version "${OS_VERSION:1}" \
+          --device "${balena_device_uuid}" \
+          --network ethernet \
+          --appUpdatePollInterval 10 \
+          --dev \
+          --output /balena/config.json
+    fi
+    cat </balena/config.json | jq -re
+
+    with_backoff balena os configure "${GUEST_IMAGE}" \
+      --fleet "${TEST_FLEET}" \
+      --version "${OS_VERSION#v}" \
+      --config-network ethernet \
+      --config /balena/config.json
+
+    touch "${GUEST_IMAGE%.*}.ready"
+}
+
+function check_device_status() {
+    if [[ -e /balena/config.json ]]; then
+        balena_device_uuid="$(cat </balena/config.json | jq -r .uuid)"
+
+        if [[ -n $balena_device_uuid ]]; then
+            is_online="$(balena devices --json --fleet "${TEST_FLEET}" \
+              | jq -r --arg uuid "${balena_device_uuid}" '.[] | select(.uuid==$uuid).is_online == true')"
+
+            if [[ $is_online =~ true ]]; then
+                return 0
+            else
+                return 1
+            fi
+        fi
+    fi
+}
+
+function wait_for_device() {
+    while ! check_device_status; do sleep "$(( (RANDOM % 5) + 5 ))s"; done
+}
+
+function registry_auth() {
+    if [[ -n $REGISTRY_USER ]] && [[ -n $REGISTRY_PASS ]]; then
+        with_backoff docker login -u "${REGISTRY_USER}" -p "${REGISTRY_PASS}"
+
+        printf '{"https://index.docker.io/v1/": {"username":"%s", "password":"$s"}}' \
+          "${REGISTRY_USER}" "${REGISTRY_PASS}" | jq -r > ~/.balena/secrets.json
+    fi
+}
+
+function deploy_release() {
+    tmpbuild="$(mktemp -d)"
+    pushd "${tmpbuild}"
+
+    echo 'FROM hello-world' >Dockerfile
+
+    while ! balena deploy \
+      --ca "${DOCKER_CERT_PATH}/ca.pem" \
+      --cert "${DOCKER_CERT_PATH}/cert.pem" \
+      --key "${DOCKER_CERT_PATH}/key.pem" \
+      "${TEST_FLEET}"; do
+
+        sleep "$(( (RANDOM % 5) + 5 ))s"
+    done
+    popd
+}
+
+function get_releases() {
+      with_backoff balena releases --json "${TEST_FLEET}"
+}
+
+function get_release_commit() {
+      echo "$(get_releases)" | jq -re \
+        'select((.[].status=="success")
+        and (.[].is_invalidated==false)
+        and (.[].is_final==true)
+        and (.[].release_type=="final"))[0].commit'
+}
+
+function get_release_id() {
+      echo "$(get_releases)" | jq -re \
+        'select((.[].status=="success")
+        and (.[].is_invalidated==false)
+        and (.[].is_final==true)
+        and (.[].release_type=="final"))[0].id'
+}
+
+function supervisor_update_target_state() {
+    local balena_device_uuid
+    balena_device_uuid="$(cat </balena/config.json | jq -r .uuid)"
+
+    if [[ -n $balena_device_uuid ]]; then
+        while ! curl ${curl_opts} "https://api.${DNS_TLD}/supervisor/v1/update" \
+          --header "Content-Type: application/json" \
+          --header "Authorization: Bearer $(cat <~/.balena/token)" \
+          --data "{\"uuid\": \"${balena_device_uuid}\", \"data\": {\"force\": true}}"; do
+
+            sleep "$(( (RANDOM % 5) + 5 ))s"
+        done
+    fi
+}
+
+function check_running_release() {
+    local balena_device_uuid
+    balena_device_uuid="$(cat </balena/config.json | jq -r .uuid)"
+
+    local should_be_running_release
+    should_be_running_release="$(get_release_commit)"
+    [[ -z $should_be_running_release ]] && false
+
+    if [[ -n $balena_device_uuid ]]; then
+        while ! [[ $(balena device "${balena_device_uuid}" | grep -E ^COMMIT | awk '{print $2}') =~ ${should_be_running_release} ]]; do
+            running_release_id="$(balena device "${balena_device_uuid}" | grep -E ^COMMIT | awk '{print $2}')"
+            printf 'please wait, device %s should be running %s, but is still running %s...\n' \
+              "${balena_device_uuid}" \
+              "${1}" \
+              "${running_release_id}"
+
+            sleep "$(( (RANDOM % 5) + 5 ))s"
+        done
+    fi
+}
+
+function get_os_version() {
+    local BALENARC_BALENA_URL
+    BALENARC_BALENA_URL="$(echo "${BALENA_API_URL}" | sed 's#https://api\.##g')"
+
+    local os_version
+    os_version=${OS_VERSION:-$(with_backoff balena os versions "${DEVICE_TYPE}" | head -n 1)}
+    echo "${os_version}"
+}
+
+function upload_release_asset() {
+    if [[ "$RELEASE_ASSETS_TEST" =~ true ]]; then
+        local release_id
+        release_id=${1:-1}
+        release_asset="$(find / -type f -name '*.png' | head -n 1)"
+
+        curl ${curl_opts} "https://api.${DNS_TLD}/resin/release_asset" \
+          --header "Authorization: Bearer $(cat <~/.balena/token)" \
+          --form "asset=@${release_asset}" \
+          --form "release=${release_id}" \
+          --form "asset_key=$((RANDOM))-$(basename ${release_asset})" \
+          | jq -re .asset.href \
+          | xargs curl ${curl_opts} -o "/tmp/$((RANDOM))-$(basename ${release_asset})"
+    fi
+}
+
+# --- main
+if [[ "$PRODUCTION_MODE" =~ true ]]; then
+    exit
+fi
+
+if [[ -n "${BALENA_DEVICE_UUID}" ]]; then
+    # prepend the device UUID if running on balenaOS
+    TLD="${BALENA_DEVICE_UUID}.${DNS_TLD}"
+else
+    TLD="${DNS_TLD}"
+fi
+
+BALENA_API_URL=${BALENA_API_URL:-https://api.balena-cloud.com}
+BALENARC_BALENA_URL="${DNS_TLD}"
+CERTS=${CERTS:-/certs}
+CONF=${CONF:-/balena/${TLD}.env}
+DEVICE_TYPE=${DEVICE_TYPE:-generic-amd64}
+GUEST_DISK_SIZE=${GUEST_DISK_SIZE:-8}
+GUEST_IMAGE=${GUEST_IMAGE:-/balena/balena.img}
+OS_VERSION="$(get_os_version)"
+TEST_FLEET=${TEST_FLEET:-test-fleet}
+
+[[ -f "$CONF" ]] && source "${CONF}"
+
+update_ca_certificates  # ensure self-signed root CA certificate(s) trust
+
+registry_auth  # optionally authenticate with DockerHub (rate-limiting)
+
+wait_for_api  # spin here until the API is responding
+
+balena whoami || open_balena_login  # spin here until authenticated
+
+create_fleet  # spin here until the fleet is created
+
+# critical section
+set_update_lock
+download_os_image
+configure_virtual_device
+deploy_release
+upload_release_asset "$(get_release_id)"  # upload an additional asset to a release
+remove_update_lock
+# .. end
+
+wait_for_device  # spin here until test-device comes online
+check_running_release  # .. and ensure the device is running our release

src/balena-tests/functions

@@ -0,0 +1,32 @@
+# https://coderwall.com/p/--eiqg/exponential-backoff-in-bash
+function with_backoff() {
+    local max_attempts=${ATTEMPTS-5}
+    local timeout=${TIMEOUT-1}
+    local attempt=0
+    local exitCode=0
+
+    set +e
+    while [[ $attempt < $max_attempts ]]
+    do
+        "$@"
+        exitCode=$?
+
+        if [[ $exitCode == 0 ]]
+        then
+          break
+        fi
+
+        echo "Failure! Retrying in $timeout.." 1>&2
+        sleep "$timeout"
+        attempt=$(( attempt + 1 ))
+        timeout=$(( timeout * 2 ))
+    done
+
+    if [[ $exitCode != 0 ]]
+    then
+        echo "You've failed me for the last time! ($*)" 1>&2
+    fi
+
+    set -e
+    return $exitCode
+}

src/cert-manager/Dockerfile

@@ -0,0 +1,4 @@
+# https://github.com/balena-io/cert-manager
+FROM balena/cert-manager:v0.2.2
+
+COPY *.json /opt/

src/cert-manager/certs.json

@@ -0,0 +1,63 @@
+[
+  {
+    "request": {
+      "key": {
+        "algo": "${key_algo}",
+        "size": ${key_size}
+      },
+      "hosts": ${hosts},
+      "names": [
+        {
+          "C": "${country}",
+          "L": "${locality_name}",
+          "O": "${org}",
+          "OU": "${org_unit}",
+          "ST": "${state}"
+        }
+      ],
+      "CN": "${TLD}"
+    }
+  },
+  {
+    "request": {
+      "key": {
+        "algo": "${key_algo}",
+        "size": ${key_size}
+      },
+      "hosts": [
+        "vpn.${TLD}"
+      ],
+      "names": [
+        {
+          "C": "${country}",
+          "L": "${locality_name}",
+          "O": "${org}",
+          "OU": "${org_unit}",
+          "ST": "${state}"
+        }
+      ],
+      "CN": "vpn.${TLD}"
+    }
+  },
+  {
+	"request": {
+	  "key": {
+		"algo": "${key_algo}",
+		"size": ${key_size}
+	  },
+	  "hosts": [
+		"api.${TLD}"
+	  ],
+	  "names": [
+		{
+		  "C": "${country}",
+		  "L": "${locality_name}",
+		  "O": "${org}",
+		  "OU": "${org_unit}",
+		  "ST": "${state}"
+		}
+	  ],
+	  "CN": "api.${TLD}"
+	}
+  }
+]

src/cert-manager/keys.json

@@ -0,0 +1 @@
+[]

src/cert-provider/Dockerfile

@@ -1,22 +0,0 @@
-FROM alpine
-
-EXPOSE 80
-WORKDIR /usr/src/app
-VOLUME [ "/usr/src/app/certs" ]
-
-RUN apk add --update bash curl git openssl ncurses socat
-
-# from https://github.com/Neilpang/acme.sh/releases/tag/2.8.5
-RUN git clone https://github.com/Neilpang/acme.sh.git && \
-    cd acme.sh && \
-    git fetch && git fetch --tags && \
-    git checkout 2.8.5 . && \
-    ./acme.sh --install  \
-    --cert-home /usr/src/app/certs
-
-COPY entry.sh /entry.sh
-COPY cert-provider.sh ./cert-provider.sh
-COPY fake-le-bundle.pem ./
-
-ENTRYPOINT [ "/entry.sh" ]
-CMD [ "/usr/src/app/cert-provider.sh" ]

src/cert-provider/cert-provider.sh

@@ -1,190 +0,0 @@
-#!/usr/bin/env bash
-
-# the acme.sh client script, installed via Git in the Dockerfile...
-ACME_BIN="$(realpath ~/.acme.sh/acme.sh)"
-
-# the path to a bundle of certs to verify a LetsEncrypt staging certificate until Apr 2036...
-ACME_STAGING_CA="/usr/src/app/fake-le-bundle.pem"
-
-# the path to a file which stores the last successful mode of certificate we acquired...
-ACME_MODE_FILE="/usr/src/app/certs/last_run_mode"
-
-# colour output helpers...
-reset=$(tput -T xterm sgr0)
-red=$(tput -T xterm setaf 1)
-green=$(tput -T xterm setaf 2)
-yellow=$(tput -T xterm setaf 3)
-blue=$(tput -T xterm setaf 4)
-
-logError() {
-    echo "${red}[Error]${reset} $1"
-}
-
-logWarn() {
-    echo "${yellow}[Warn]${reset} $1"
-}
-
-logInfo() {
-    echo "${blue}[Info]${reset} $1"
-}
-
-logSuccess() {
-    echo "${green}[Success]${reset} $1"
-}
-
-logErrorAndStop() {
-    logError "$1 [Stopping]"
-    while true; do
-        # do nothing forever...
-        sleep 60
-    done
-}
-
-retryWithDelay() {
-    RETRIES=${2:-3}
-    DELAY=${3:-5}
-
-    local ATTEMPT=0
-    while [ "$RETRIES" -gt "$ATTEMPT" ]; do
-        (( ATTEMPT++ ))
-        logInfo "($ATTEMPT/$RETRIES) Connecting..."
-        if $1; then
-            logInfo "($ATTEMPT/$RETRIES) Success!"
-            return $?
-        fi
-
-        if [ "$RETRIES" -gt "$ATTEMPT" ]; then
-            logInfo "($ATTEMPT/$RETRIES) Failed. Retrying in ${DELAY} seconds..."
-            sleep "$DELAY"
-        else
-            logInfo "($ATTEMPT/$RETRIES) Failed!"
-        fi
-    done
-
-    return 1
-}
-
-waitForOnline() {
-    ADDRESS="${1,,}"
-
-    logInfo "Waiting for ${ADDRESS} to be available via HTTP..."
-    retryWithDelay "curl --output /dev/null --silent --head --fail --max-time 5 http://${ADDRESS}"
-}
-
-isUsingStagingCert() {
-    HOST="${1,,}"
-    echo "" | openssl s_client -host "$HOST" -port 443 -showcerts 2>/dev/null | awk '/BEGIN CERT/ {p=1} ; p==1; /END CERT/ {p=0}' | openssl verify -CAfile "$ACME_STAGING_CA" > /dev/null 2>&1
-}
-
-pre-flight() {
-    case "$ACTIVE" in
-        "true"|"yes")
-            ;;
-        *)
-            logError "ACTIVE variable is not enabled. Value should be \"true\" or \"yes\" to continue."
-            return 1
-            ;;
-    esac
-
-    if [ -z "$DOMAINS" ]; then
-        logError "DOMAINS must be set. Value should be a comma-delimited string of domains."
-        return 1
-    else
-        IFS=, read -r -a ACME_DOMAINS <<< "$DOMAINS"
-        IFS=' ' read -r -a ACME_DOMAIN_ARGS <<< "${ACME_DOMAINS[@]/#/-d }"
-    fi
-
-    if [ -z "$VALIDATION" ]; then
-        logInfo "VALIDATION not set. Using default: http-01"
-        VALIDATION="http-01"
-    else
-        case "$VALIDATION" in
-            "http-01")
-                logInfo "Using validation method: $VALIDATION"
-                ;;
-            *)
-                logError "VALIDATION is invalid. Use a valid value: http-01"
-                return 1
-                ;;
-        esac
-    fi
-
-    if [ -z "$OUTPUT_PEM" ]; then
-        logError "OUTPUT_PEM must be set. Value should be the path to install your certificate to."
-        return 1
-    fi
-}
-
-waitToSeeStagingCert() {
-    logInfo "Waiting for ${ACME_DOMAINS[0]} to use a staging certificate..."
-    retryWithDelay "isUsingStagingCert ${ACME_DOMAINS[0]}" 3 5
-}
-
-lastAcquiredCertFor() {
-    ACME_MODE="${1:-none}"
-    ACME_LAST_MODE="$(cat $ACME_MODE_FILE || echo '')"
-    logInfo "Last acquired certificate for ${ACME_LAST_MODE^^}"
-    [ "${ACME_LAST_MODE,,}" == "${ACME_MODE,,}" ]
-}
-
-acquireCertificate() {
-    ACME_MODE="${1:-staging}"
-    ACME_FORCE="${2:-false}"
-    ACME_OPTS=()
-
-    if [ "${ACME_FORCE,,}" == "true" ];then ACME_OPTS+=("--force"); fi
-    case "$ACME_MODE" in
-        "production")
-            logInfo "Using PRODUCTION mode"
-            ;;
-        *)
-            logInfo "Using STAGING mode"
-            ACME_OPTS+=("--staging")
-            ;;
-    esac
-
-    case "$VALIDATION" in
-        "http-01")
-            ACME_OPTS+=("--standalone")
-            ;;
-        *)
-            logError "VALIDATION is invalid. Use a valid value: http-01"
-            return 1
-            ;;
-    esac
-
-    if ! waitForOnline "${ACME_DOMAINS[0]}"; then
-        logError "Unable to access site over HTTP"
-        return 1
-    fi
-
-    logInfo "Issuing certificates..."
-    "$ACME_BIN" --issue "${ACME_OPTS[@]}" "${ACME_DOMAIN_ARGS[@]}"
-
-    logInfo "Installing certificates..." && \
-    "$ACME_BIN" --install-cert "${ACME_DOMAIN_ARGS[@]}" \
-    --cert-file      /tmp/cert.pem  \
-    --key-file       /tmp/key.pem  \
-    --fullchain-file /tmp/fullchain.pem \
-    --reloadcmd     "cat /tmp/fullchain.pem /tmp/key.pem > $OUTPUT_PEM" && \
-
-    echo "${ACME_MODE}" > "${ACME_MODE_FILE}"
-}
-
-pre-flight || logErrorAndStop "Unable to continue due to misconfiguration. See errors above."
-
-while ! waitForOnline "${ACME_DOMAINS[0]}"; do
-    logInfo "Unable to access ${ACME_DOMAINS[0]} on port 80. This is needed for certificate validation. Retrying in 30 seconds..."
-    sleep 30
-done
-
-if ! lastAcquiredCertFor "production"; then
-    acquireCertificate "staging" || logErrorAndStop "Unable to acquire a staging certificate."
-    waitToSeeStagingCert || logErrorAndStop "Unable to detect certificate change over. Cannot issue a production certificate."
-    acquireCertificate "production" "true" || logErrorAndStop "Unable to acquire a production certificate."
-fi
-
-logSuccess "Done!"
-
-logInfo "Running cron..."
-crond -f -d 7

src/cert-provider/entry.sh

@@ -1,3 +0,0 @@
-#!/bin/bash
-
-exec "$@"

src/cert-provider/fake-le-bundle.pem

@@ -1,56 +0,0 @@
------BEGIN CERTIFICATE-----
-MIIFATCCAumgAwIBAgIRAKc9ZKBASymy5TLOEp57N98wDQYJKoZIhvcNAQELBQAw
-GjEYMBYGA1UEAwwPRmFrZSBMRSBSb290IFgxMB4XDTE2MDMyMzIyNTM0NloXDTM2
-MDMyMzIyNTM0NlowGjEYMBYGA1UEAwwPRmFrZSBMRSBSb290IFgxMIICIjANBgkq
-hkiG9w0BAQEFAAOCAg8AMIICCgKCAgEA+pYHvQw5iU3v2b3iNuYNKYgsWD6KU7aJ
-diddtZQxSWYzUI3U0I1UsRPTxnhTifs/M9NW4ZlV13ZfB7APwC8oqKOIiwo7IwlP
-xg0VKgyz+kT8RJfYr66PPIYP0fpTeu42LpMJ+CKo9sbpgVNDZN2z/qiXrRNX/VtG
-TkPV7a44fZ5bHHVruAxvDnylpQxJobtCBWlJSsbIRGFHMc2z88eUz9NmIOWUKGGj
-EmP76x8OfRHpIpuxRSCjn0+i9+hR2siIOpcMOGd+40uVJxbRRP5ZXnUFa2fF5FWd
-O0u0RPI8HON0ovhrwPJY+4eWKkQzyC611oLPYGQ4EbifRsTsCxUZqyUuStGyp8oa
-aoSKfF6X0+KzGgwwnrjRTUpIl19A92KR0Noo6h622OX+4sZiO/JQdkuX5w/HupK0
-A0M0WSMCvU6GOhjGotmh2VTEJwHHY4+TUk0iQYRtv1crONklyZoAQPD76hCrC8Cr
-IbgsZLfTMC8TWUoMbyUDgvgYkHKMoPm0VGVVuwpRKJxv7+2wXO+pivrrUl2Q9fPe
-Kk055nJLMV9yPUdig8othUKrRfSxli946AEV1eEOhxddfEwBE3Lt2xn0hhiIedbb
-Ftf/5kEWFZkXyUmMJK8Ra76Kus2ABueUVEcZ48hrRr1Hf1N9n59VbTUaXgeiZA50
-qXf2bymE6F8CAwEAAaNCMEAwDgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQFMAMB
-Af8wHQYDVR0OBBYEFMEmdKSKRKDm+iAo2FwjmkWIGHngMA0GCSqGSIb3DQEBCwUA
-A4ICAQBCPw74M9X/Xx04K1VAES3ypgQYH5bf9FXVDrwhRFSVckria/7dMzoF5wln
-uq9NGsjkkkDg17AohcQdr8alH4LvPdxpKr3BjpvEcmbqF8xH+MbbeUEnmbSfLI8H
-sefuhXF9AF/9iYvpVNC8FmJ0OhiVv13VgMQw0CRKkbtjZBf8xaEhq/YqxWVsgOjm
-dm5CAQ2X0aX7502x8wYRgMnZhA5goC1zVWBVAi8yhhmlhhoDUfg17cXkmaJC5pDd
-oenZ9NVhW8eDb03MFCrWNvIh89DDeCGWuWfDltDq0n3owyL0IeSn7RfpSclpxVmV
-/53jkYjwIgxIG7Gsv0LKMbsf6QdBcTjhvfZyMIpBRkTe3zuHd2feKzY9lEkbRvRQ
-zbh4Ps5YBnG6CKJPTbe2hfi3nhnw/MyEmF3zb0hzvLWNrR9XW3ibb2oL3424XOwc
-VjrTSCLzO9Rv6s5wi03qoWvKAQQAElqTYRHhynJ3w6wuvKYF5zcZF3MDnrVGLbh1
-Q9ePRFBCiXOQ6wPLoUhrrbZ8LpFUFYDXHMtYM7P9sc9IAWoONXREJaO08zgFtMp4
-8iyIYUyQAbsvx8oD2M8kRvrIRSrRJSl6L957b4AFiLIQ/GgV2curs0jje7Edx34c
-idWw1VrejtwclobqNMVtG3EiPUIpJGpbMcJgbiLSmKkrvQtGng==
------END CERTIFICATE-----
------BEGIN CERTIFICATE-----
-MIIEqzCCApOgAwIBAgIRAIvhKg5ZRO08VGQx8JdhT+UwDQYJKoZIhvcNAQELBQAw
-GjEYMBYGA1UEAwwPRmFrZSBMRSBSb290IFgxMB4XDTE2MDUyMzIyMDc1OVoXDTM2
-MDUyMzIyMDc1OVowIjEgMB4GA1UEAwwXRmFrZSBMRSBJbnRlcm1lZGlhdGUgWDEw
-ggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDtWKySDn7rWZc5ggjz3ZB0
-8jO4xti3uzINfD5sQ7Lj7hzetUT+wQob+iXSZkhnvx+IvdbXF5/yt8aWPpUKnPym
-oLxsYiI5gQBLxNDzIec0OIaflWqAr29m7J8+NNtApEN8nZFnf3bhehZW7AxmS1m0
-ZnSsdHw0Fw+bgixPg2MQ9k9oefFeqa+7Kqdlz5bbrUYV2volxhDFtnI4Mh8BiWCN
-xDH1Hizq+GKCcHsinDZWurCqder/afJBnQs+SBSL6MVApHt+d35zjBD92fO2Je56
-dhMfzCgOKXeJ340WhW3TjD1zqLZXeaCyUNRnfOmWZV8nEhtHOFbUCU7r/KkjMZO9
-AgMBAAGjgeMwgeAwDgYDVR0PAQH/BAQDAgGGMBIGA1UdEwEB/wQIMAYBAf8CAQAw
-HQYDVR0OBBYEFMDMA0a5WCDMXHJw8+EuyyCm9Wg6MHoGCCsGAQUFBwEBBG4wbDA0
-BggrBgEFBQcwAYYoaHR0cDovL29jc3Auc3RnLXJvb3QteDEubGV0c2VuY3J5cHQu
-b3JnLzA0BggrBgEFBQcwAoYoaHR0cDovL2NlcnQuc3RnLXJvb3QteDEubGV0c2Vu
-Y3J5cHQub3JnLzAfBgNVHSMEGDAWgBTBJnSkikSg5vogKNhcI5pFiBh54DANBgkq
-hkiG9w0BAQsFAAOCAgEABYSu4Il+fI0MYU42OTmEj+1HqQ5DvyAeyCA6sGuZdwjF
-UGeVOv3NnLyfofuUOjEbY5irFCDtnv+0ckukUZN9lz4Q2YjWGUpW4TTu3ieTsaC9
-AFvCSgNHJyWSVtWvB5XDxsqawl1KzHzzwr132bF2rtGtazSqVqK9E07sGHMCf+zp
-DQVDVVGtqZPHwX3KqUtefE621b8RI6VCl4oD30Olf8pjuzG4JKBFRFclzLRjo/h7
-IkkfjZ8wDa7faOjVXx6n+eUQ29cIMCzr8/rNWHS9pYGGQKJiY2xmVC9h12H99Xyf
-zWE9vb5zKP3MVG6neX1hSdo7PEAb9fqRhHkqVsqUvJlIRmvXvVKTwNCP3eCjRCCI
-PTAvjV+4ni786iXwwFYNz8l3PmPLCyQXWGohnJ8iBm+5nk7O2ynaPVW0U2W+pt2w
-SVuvdDM5zGv2f9ltNWUiYZHJ1mmO97jSY/6YfdOUH66iRtQtDkHBRdkNBsMbD+Em
-2TgBldtHNSJBfB3pm9FblgOcJ0FSWcUDWJ7vO0+NTXlgrRofRT6pVywzxVo6dND0
-WzYlTWeUVsO40xJqhgUQRER9YLOLxJ0O6C8i0xFxAMKOtSdodMB3RIwt7RFQ0uyt
-n5Z5MqkYhlMI3J1tPRTp1nEt9fyGspBOO05gi148Qasp+3N+svqKomoQglNoAxU=
------END CERTIFICATE-----

src/haproxy-sidecar/Dockerfile

@@ -0,0 +1,7 @@
+FROM docker
+
+COPY balena.sh /usr/local/bin/balena.sh
+
+RUN apk add --no-cache bash
+
+CMD /usr/local/bin/balena.sh

src/haproxy-sidecar/balena.sh

@@ -0,0 +1,75 @@
+#!/usr/bin/env bash
+
+set -ea
+
+[[ $VERBOSE =~ on|On|Yes|yes|true|True ]] && set -x
+
+function cleanup() {
+   rm -f /host/run/docker.sock
+
+   # crash loop backoff
+   sleep "$(( (RANDOM % 5) + 5 ))s"
+}
+
+trap 'cleanup' EXIT
+
+if [ -S /host/run/balena-engine.sock ]; then
+    ln -s /host/run/balena-engine.sock /host/run/docker.sock
+fi
+
+which curl || apk add curl --no-cache
+which jq || apk add jq --no-cache
+
+if docker inspect "${BALENA_APP_UUID}_default" --format "{{.ID}}"; then
+    network="${BALENA_APP_UUID}_default"
+elif docker inspect "${BALENA_APP_ID}_default" --format "{{.ID}}"; then
+    network="${BALENA_APP_ID}_default"
+else
+    network=open-balena_default
+fi
+
+# shellcheck disable=SC2153
+for alias in ${ALIASES//,/ }; do
+    hostname="${alias}.${DNS_TLD}"
+    aliases="--alias ${hostname} ${aliases}"
+done
+
+while true; do
+    if [[ -n $BALENA_SUPERVISOR_ADDRESS ]] && [[ -n $BALENA_SUPERVISOR_API_KEY ]]; then
+        while [[ "$(curl --silent --retry 3 --fail \
+          "${BALENA_SUPERVISOR_ADDRESS}/v1/device?apikey=${BALENA_SUPERVISOR_API_KEY}" \
+          -H "Content-Type:application/json" | jq -r '.update_pending')" =~ true ]]; do
+            sleep "$(( (RANDOM % 3) + 3 ))s"
+        done
+        sleep "$(( (RANDOM % 5) + 5 ))s"
+    fi
+
+    while [[ "$(docker ps \
+      --filter "name=haproxy" \
+      --filter "expose=1936/tcp" \
+      --filter "status=running" \
+      --filter "network=${network}" \
+      --format "{{.ID}}")" == '' ]]; do
+        sleep "$(( (RANDOM % 3) + 3 ))s"
+    done
+
+    haproxy="$(docker ps \
+      --filter "name=haproxy" \
+      --filter "expose=1936/tcp" \
+      --filter "status=running" \
+      --filter "network=${network}" \
+      --format "{{.ID}}")"
+
+    if ! [[ $restarted == "${haproxy}" ]]; then
+        docker network disconnect "${network}" "${haproxy}"
+
+        # shellcheck disable=SC2086
+        docker network connect --alias haproxy ${aliases} "${network}" "${haproxy}"
+
+        docker restart "${haproxy}"
+
+        restarted="${haproxy}"
+    fi
+
+    sleep "$(( (RANDOM % 15) + 15 ))s"
+done

src/haproxy/Dockerfile

@@ -1,10 +1,4 @@
-FROM haproxy:1.9-alpine
-
-VOLUME [ "/certs" ]
-
-RUN apk add --update inotify-tools
+# https://github.com/balena-io/open-balena-haproxy
+FROM balena/open-balena-haproxy:v4.3.1
 
 COPY haproxy.cfg /usr/local/etc/haproxy/haproxy.cfg
-COPY start-haproxy.sh /start-haproxy
-
-CMD /start-haproxy

src/haproxy/haproxy.cfg

@@ -1,134 +1,176 @@
 global
-  tune.ssl.default-dh-param 1024
+	tune.ssl.default-dh-param 1024
+	# https://github.com/haproxytech/haproxy-lua-cors
+	lua-load /usr/local/etc/haproxy/cors.lua
+	# https://www.haproxy.com/blog/introduction-to-haproxy-logging/
+	log stdout format raw daemon "${LOGLEVEL}"
+	log stderr format raw daemon "${LOGLEVEL}"
+	ssl-default-bind-options ssl-min-ver TLSv1.2
 
 defaults
-  timeout connect 5s
-  timeout client 50s
-  timeout server 50s
+	balance roundrobin
+	default-server init-addr last,libc,none
+	default-server inter 3s rise 2 fall 3
+	log global
+	mode http
+	option contstats
+	option dontlognull
+	option forwardfor
+	option httplog
+	timeout client 63s
+	timeout connect 5s
+	timeout http-keep-alive 1s
+	timeout http-request 63s
+	timeout server 63s
+	timeout tunnel 3600s
 
-frontend http-in
-  mode http
-  option forwardfor
-  bind *:80
-  reqadd X-Forwarded-Proto:\ http
+resolvers docker-bridge-resolver
+	nameserver docker-resolver 127.0.0.11:53
+	hold valid 0ms
 
-  acl is_cert_validation path -i -m beg "/.well-known/acme-challenge/"
-  use_backend cert-provider if is_cert_validation
+http-errors balena-http-errors
+	errorfile 400 /etc/haproxy/errors/400.http
+	errorfile 401 /etc/haproxy/errors/401.http
+	errorfile 403 /etc/haproxy/errors/403.http
+	errorfile 404 /etc/haproxy/errors/404.http
+	errorfile 500 /etc/haproxy/errors/500.http
+	errorfile 502 /etc/haproxy/errors/502.http
+	errorfile 503 /etc/haproxy/errors/503.http
 
-  acl host_api hdr_dom(host) -i "api.${HAPROXY_HOSTNAME}"
-  use_backend backend_api if host_api
+userlist balena
+	user balena insecure-password "${BALENA_DEVICE_UUID}"
 
-  acl host_registry hdr_dom(host) -i "registry.${HAPROXY_HOSTNAME}"
-  use_backend backend_registry if host_registry
+listen haproxy-stats
+	bind :::1936 v4v6 ssl crt "${CERT_CHAIN_PATH}" alpn h2,http/1.1
+	stats auth "balena:${BALENA_DEVICE_UUID}"
+	stats enable
+	stats uri /metrics
 
-  acl host_vpn hdr_dom(host) -i "vpn.${HAPROXY_HOSTNAME}"
-  use_backend backend_vpn if host_vpn
+frontend http
+	bind :::80 v4v6
+	default_backend api-backend
+	errorfiles balena-http-errors
+	http-request capture req.hdr(Host) len 15
+	http-response lua.cors
+	# https://www.haproxy.com/blog/haproxy-log-customization/
+	log-format "%ci:%cp [%tr] %ft %b/%s %TR/%Tw/%Tc/%Tr/%Ta %ST %B %CC %CS %tsc %ac/%fc/%bc/%sc/%rc %sq/%bq %hr %hs %{+Q}r"
 
-  acl host_s3 hdr_dom(host) -i "s3.${HAPROXY_HOSTNAME}"
-  use_backend backend_s3 if host_s3
+	acl api_dead nbsrv(api-backend) lt 1
+	acl registry_dead nbsrv(registry-backend) lt 1
+	acl vpn_dead nbsrv(vpn-backend) lt 1
+	monitor-uri /health
+	monitor fail if api_dead registry_dead vpn_dead
 
-frontend ssl-in
-  mode tcp
-  bind *:443
-  tcp-request inspect-delay 2s
-  tcp-request content accept if { req.ssl_hello_type 1 }
+	acl host-api-backend hdr_beg(host) -i "api."
+	# default public device URL(s) always go to the API
+	acl host-pdu-default hdr(host) -m reg -i "\.?([0-9a-f]{32}|${BALENA_DEVICE_UUID})\.(devices|balena-?(.*)-devices)\."
+	use_backend api-backend if host-api-backend || host-pdu-default
 
-  acl is_ssl req.ssl_ver 2:3.4
+	acl host-registry-backend hdr_beg(host) -i "registry2."
+	http-request add-header X-Forwarded-Proto http if host-registry-backend
+	use_backend registry-backend if host-registry-backend
 
-  acl host_tunnel req_ssl_sni -i "tunnel.${HAPROXY_HOSTNAME}"
-  use_backend redirect-to-tunnel-in if host_tunnel
+	acl host-s3-backend hdr_beg(host) -i "s3."
+	http-request add-header X-Forwarded-Proto http if host-s3-backend
+	use_backend s3-backend if host-s3-backend
 
-  use_backend redirect-to-https-in if is_ssl
-  use_backend vpn-devices if !is_ssl
+	acl host-minio-backend hdr_beg(host) -i "minio."
+	http-request add-header X-Forwarded-Proto http if host-minio-backend
+	use_backend minio-backend if host-minio-backend
 
-backend redirect-to-https-in
-  mode tcp
-  balance roundrobin
-  server localhost 127.0.0.1:444 send-proxy-v2
+# routes between OpenVPN, SSL and HTTPS traffic
+frontend tcp-router
+	mode tcp
+	option tcplog
+	log-format "%ci:%cp [%t] %ft %b/%s %Tw/%Tc/%Tt %B %ts %ac/%fc/%bc/%sc/%rc %sq/%bq"
+	bind :::443 v4v6
+	tcp-request inspect-delay 2s
+	tcp-request content accept if { req.ssl_hello_type 1 }
+	acl is_ssl req.ssl_ver 2:3.4
 
-backend redirect-to-tunnel-in
-  mode tcp
-  balance roundrobin
-  server localhost 127.0.0.1:3129
+	acl sni-host-tunnel req_ssl_sni -m beg "tunnel."
+	use_backend redirect-to-tunnel if sni-host-tunnel
 
-frontend https-in
-  mode http
-  option forwardfor
-  bind 127.0.0.1:444 ssl crt /etc/ssl/private/open-balena.pem accept-proxy
-  reqadd X-Forwarded-Proto:\ https
+	# everything else => HTTPS
+	use_backend redirect-to-https if is_ssl
 
-  acl host_api hdr_dom(host) -i "api.${HAPROXY_HOSTNAME}"
-  use_backend backend_api if host_api
+	# or VPN
+	use_backend vpn-backend if !is_ssl
 
-  acl host_registry hdr_dom(host) -i "registry.${HAPROXY_HOSTNAME}"
-  use_backend backend_registry if host_registry
+backend redirect-to-tunnel
+	mode tcp
+	server localhost 127.0.0.1:3129 send-proxy-v2
 
-  acl host_vpn hdr_dom(host) -i "vpn.${HAPROXY_HOSTNAME}"
-  use_backend backend_vpn if host_vpn
+# https://stackoverflow.com/a/39213442/1559300
+listen tunnel-backend
+	mode tcp
+	option tcplog
+	log-format "%ci:%cp [%t] %ft %b/%s %Tw/%Tc/%Tt %B %ts %ac/%fc/%bc/%sc/%rc %sq/%bq"
+	bind 127.0.0.1:3129 ssl crt "${CERT_CHAIN_PATH}" alpn h2,http/1.1 accept-proxy
+	server tunnel vpn:3128 resolvers docker-bridge-resolver resolve-prefer ipv4 check port 3128
 
-  acl host_s3 hdr_dom(host) -i "s3.${HAPROXY_HOSTNAME}"
-  use_backend backend_s3 if host_s3
+backend redirect-to-https
+	mode tcp
+	server localhost 127.0.0.1:444 send-proxy-v2
 
-backend backend_api
-  mode http
-  option forwardfor
-  balance roundrobin
-  server balena_api_1 api:80 check port 80
+frontend https
+	bind 127.0.0.1:444 ssl crt "${CERT_CHAIN_PATH}" alpn h2,http/1.1 accept-proxy
+	default_backend api-backend
+	errorfiles balena-http-errors
+	http-request add-header X-Forwarded-Proto https
+	http-request capture req.hdr(Host) len 15
+	http-response lua.cors
+	log-format "%ci:%cp [%tr] %ft %b/%s %TR/%Tw/%Tc/%Tr/%Ta %ST %B %CC %CS %tsc %ac/%fc/%bc/%sc/%rc %sq/%bq %hr %hs %{+Q}r"
 
-backend backend_registry
-  mode http
-  option forwardfor
-  balance roundrobin
-  server balena_registry_1 registry:80 check port 80
+	acl host-api-backend hdr_beg(host) -i "api."
+	use_backend api-backend if host-api-backend
 
-backend backend_vpn
-  mode http
-  option forwardfor
-  balance roundrobin
-  server balena_vpn_1 vpn:80 check port 80
+	acl host-registry-backend hdr_beg(host) -i "registry2."
+	use_backend registry-backend if host-registry-backend
 
-backend backend_s3
-  mode http
-  option forwardfor
-  balance roundrobin
-  server balena_s3_1 s3:80 check port 80
+	acl host-s3-backend hdr_beg(host) -i "s3."
+	use_backend s3-backend if host-s3-backend
 
-backend cert-provider
-  mode http
-  option forwardfor
-  balance roundrobin
-  server balena_cert-provider_1 cert-provider:80 no-check
+	acl host-minio-backend hdr_beg(host) -i "minio."
+	use_backend minio-backend if host-minio-backend
 
-backend vpn-devices
-  mode tcp
-  server balena_vpn_1 vpn:443 send-proxy-v2 check-send-proxy port 443
+	acl host-ca-backend hdr_beg(host) -i "ca."
+	# only allow CRL requests unauthenticated, protect everything else
+	acl balena-ca-crl path -i -m beg /api/v1/cfssl/crl
+	acl balena-ca-auth http_auth(balena)
+	http-request auth realm balena-ca if host-ca-backend !balena-ca-auth !balena-ca-crl
+	use_backend ca-backend if host-ca-backend
 
-frontend db
-  mode tcp
-  bind *:5432
-  default_backend backend_db
-  timeout client 1h
+	acl host-ocsp-backend hdr_beg(host) -i "ocsp."
+	use_backend ocsp-backend if host-ocsp-backend
 
-backend backend_db
-  mode tcp
-  server balena_db_1 db:5432 check port 5432
+backend api-backend
+	server api api:80 resolvers docker-bridge-resolver resolve-prefer ipv4 check port 80
 
-frontend redis
-  mode tcp
-  bind *:6379
-  default_backend backend_redis
-  timeout client 1h
+backend registry-backend
+	server registry registry:80 resolvers docker-bridge-resolver resolve-prefer ipv4 check port 80
 
-backend backend_redis
-  mode tcp
-  server balena_redis_1 redis:6379 check port 6379
+backend s3-backend
+	server s3 s3:80 resolvers docker-bridge-resolver resolve-prefer ipv4 check port 80
 
-listen vpn-tunnel
-  mode tcp
-  bind *:3128
-  server balena_vpn vpn:3128 check port 3128
+# https://github.com/minio/console
+backend minio-backend
+	server s3-console s3:43697 resolvers docker-bridge-resolver resolve-prefer ipv4 check port 43697
 
-listen vpn-tunnel-tls
-  mode tcp
-  bind *:3129 ssl crt /etc/ssl/private/open-balena.pem
-  server balena_vpn vpn:3128 check port 3128
+backend db-backend
+	mode tcp
+	server db db:5432 resolvers docker-bridge-resolver resolve-prefer ipv4 check port 5432
+
+backend redis-backend
+	mode tcp
+	server redis redis:6379 resolvers docker-bridge-resolver resolve-prefer ipv4 check port 6379
+
+backend ca-backend
+	server cfssl-ca balena-ca:8888 resolvers docker-bridge-resolver resolve-prefer ipv4 check port 8888
+
+backend ocsp-backend
+	server cfssl-ocsp balena-ca:8889 resolvers docker-bridge-resolver resolve-prefer ipv4 check port 8889
+
+backend vpn-backend
+	mode tcp
+	server openvpn vpn:443 resolvers docker-bridge-resolver resolve-prefer ipv4 send-proxy-v2 check-send-proxy check port 443

src/haproxy/start-haproxy.sh

@@ -1,32 +0,0 @@
-#!/bin/sh
-
-OPENBALENA_CERT=/etc/ssl/private/open-balena.pem
-mkdir -p "$(dirname "${OPENBALENA_CERT}")"
-
-if [ -f "/certs/open-balena.pem" ]; then
-    echo "Using certificate from cert-provider..."
-    cp /certs/open-balena.pem "${OPENBALENA_CERT}"
-else
-    echo "Building certificate from environment variables..."
-    (
-        echo "${BALENA_HAPROXY_CRT}" | base64 -d
-        echo "${BALENA_HAPROXY_KEY}" | base64 -d
-        echo "${BALENA_ROOT_CA}" | base64 -d
-    ) > "${OPENBALENA_CERT}"
-fi
-
-haproxy -f /usr/local/etc/haproxy/haproxy.cfg -W &
-HAPROXY_PID=$!
-
-while true; do
-    inotifywait -r -e create -e modify -e delete /certs
-    
-    if [ -f "/certs/open-balena.pem" ]; then
-        echo "Updating certificate from cert-provider..."
-        cp /certs/open-balena.pem "${OPENBALENA_CERT}"
-    fi
-    
-    echo "Certificate change detected. Reloading..."
-    kill -SIGUSR2 $HAPROXY_PID
-    sleep 1;
-done

src/tag-sidecar/Dockerfile

@@ -0,0 +1,5 @@
+FROM bash:alpine3.19
+
+COPY balena.sh /usr/local/bin/balena.sh
+
+CMD /usr/local/bin/balena.sh

src/tag-sidecar/balena.sh

@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+
+set -ea
+
+[[ $VERBOSE =~ on|On|Yes|yes|true|True ]] && set -x
+
+[[ $ENABLED == 'true' ]] || exit
+
+curl_with_opts() {
+    curl --fail --silent --retry 3 --connect-timeout 3 --compressed "$@"
+}
+
+get_aws_meta() {
+    if [[ $1 =~ ^.*/$ ]]; then
+        for key in $(curl_with_opts "$1"); do
+            get_aws_meta "$1${key}"
+        done
+    else
+        echo "$(echo "$1" | cut -c41-);$(curl_with_opts "$1" | tr '\n' ',')"
+    fi
+}
+
+if [[ -n $BALENA_API_URL ]] && [[ -n $BALENA_DEVICE_UUID ]] && [[ -n $BALENA_API_KEY ]]; then
+	which curl || apk add curl --no-cache
+	which jq || apk add jq --no-cache
+
+	device_id="$(curl_with_opts \
+	  "${BALENA_API_URL}/v6/device?\$filter=uuid%20eq%20'${BALENA_DEVICE_UUID}'" \
+	  -H "Content-Type: application/json" \
+	  -H "Authorization: Bearer ${BALENA_API_KEY}" | jq -r .d[].id)"
+
+	for key in $(curl_with_opts http://169.254.169.254/latest/meta-data \
+	  | grep -Ev 'iam|metrics|identity-credentials|network|events'); do
+		for kv in $(get_aws_meta "http://169.254.169.254/latest/meta-data/${key}"); do
+			tag_key="$(echo "${kv}" | awk -F';' '{print $1}')"
+			value="$(echo "${kv}" | awk -F';' '{print $2}')"
+
+			curl_with_opts "${BALENA_API_URL}/v6/device_tag" \
+			  -H "Content-Type: application/json" \
+			  -H "Authorization: Bearer ${BALENA_API_KEY}" \
+			  --data "{\"device\":\"${device_id}\",\"tag_key\":\"${tag_key}\",\"value\":\"${value}\"}"
+		done
+	done
+fi

src/test-device/Dockerfile

@@ -0,0 +1,11 @@
+# https://hub.docker.com/r/qemux/qemu-docker
+# https://github.com/qemus/qemu-docker
+FROM qemux/qemu-docker:4.24
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+	minicom \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY balena.sh /usr/sbin/
+
+WORKDIR /balena

src/test-device/balena.sh

@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+
+set -ae
+
+[[ $VERBOSE =~ on|On|Yes|yes|true|True ]] && set -x
+
+function cleanup() {
+   rm -f "${tmpimg}"
+
+   # crash loop backoff
+   sleep "$(( (RANDOM % 5) + 5 ))s"
+}
+trap 'cleanup' EXIT
+
+if [[ "$PRODUCTION_MODE" =~ true ]]; then
+    exit
+fi
+
+until test -f "${GUEST_IMAGE%.*}.ready"; do sleep "$(( (RANDOM % 5) + 5 ))s"; done
+
+tmpimg="$(mktemp)"
+cat <"${GUEST_IMAGE}" >"${tmpimg}"
+
+exec /usr/bin/qemu-system-x86_64 \
+  -bios /usr/share/ovmf/OVMF.fd \
+  -chardev socket,id=serial0,path=/run/console.sock,server=on,wait=off \
+  -cpu max \
+  -device ahci,id=ahci \
+  -device ide-hd,drive=disk,bus=ahci.0 \
+  -device virtio-net-pci,netdev=n1 \
+  -drive file="${tmpimg}",media=disk,cache=none,format=raw,if=none,id=disk \
+  -m "${MEMORY}" \
+  -machine q35 \
+  -netdev "user,id=n1,dns=127.0.0.1,guestfwd=tcp:10.0.2.100:80-cmd:netcat haproxy 80,guestfwd=tcp:10.0.2.100:443-cmd:netcat haproxy 443" \
+  -nodefaults \
+  -nographic \
+  -serial chardev:serial0 \
+  -smp "${CPU}"