v4.0.4

Update balena/open-balena-registry Docker tag to v2.39.56

.github/workflows/flowzone.yml

@@ -0,0 +1,57 @@
+name: Flowzone
+
+on:
+  pull_request:
+    types: [opened, synchronize, closed]
+    branches: [main, master]
+  pull_request_target:
+    types: [opened, synchronize, closed]
+    branches: [main, master]
+
+jobs:
+  flowzone:
+    name: Flowzone
+    uses: product-os/flowzone/.github/workflows/flowzone.yml@master
+    # prevent duplicate workflow executions for pull_request and pull_request_target
+    if: |
+      (
+        github.event.pull_request.head.repo.full_name == github.repository &&
+        github.event_name == 'pull_request'
+      ) || (
+        github.event.pull_request.head.repo.full_name != github.repository &&
+        github.event_name == 'pull_request_target'
+      )
+    secrets: inherit
+    with:
+      jobs_timeout_minutes: 60
+      cloudflare_website: open-balena
+      custom_runs_on: |
+        [
+          [
+            "self-hosted",
+            "Linux",
+            "X64"
+          ]
+        ]
+
+      balena_slugs: |
+        balena/open-balena
+
+  tests:
+    name: openBalena tests
+    uses: ./.github/workflows/tests.yml
+    needs: [flowzone]
+    if: |
+      ((
+        github.event.pull_request.head.repo.full_name == github.repository &&
+        github.event_name == 'pull_request'
+      ) || (
+        github.event.pull_request.head.repo.full_name != github.repository &&
+        github.event_name == 'pull_request_target'
+      )) && github.event.action != 'closed'
+    secrets: inherit
+    with:
+      environment: balena-cloud.com
+      fleet: balena/open-balena
+      # https://dash.cloudflare.com/001b3ed2352612aaa068aca1b0022736/balena-devices.com/dns
+      dns_tld: balena-devices.com

.github/workflows/tests.yml

@@ -0,0 +1,676 @@
+---
+name: openBalena tests
+
+on:
+  workflow_call:
+    inputs:
+      environment:
+        description: "balenaCloud environment"
+        required: true
+        type: string
+      fleet:
+        description: "balenaCloud fleet"
+        required: true
+        type: string
+      dns_tld:
+        description: "domain name to use for issuing SSL certificates"
+        required: true
+        type: string
+
+# https://docs.github.com/en/actions/security-guides/automatic-token-authentication
+# https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#permissions
+permissions:
+  actions: read
+  checks: read
+  contents: read
+  deployments: read
+  id-token: write # AWS GitHub OIDC required: write
+  issues: read
+  discussions: read
+  packages: read
+  pages: read
+  pull-requests: read
+  repository-projects: read
+  security-events: read
+  statuses: read
+
+env:
+  # Stack ID
+  # arn:aws:cloudformation:us-east-1:491725000532:stack/balena-tests-s3-certs/814dea60-404d-11ed-b06f-0a7d458f8ba5
+  AWS_S3_CERTS_BUCKET: balena-tests-certs
+  # (kvm) nested virtualisation not supported on AWS/EC2 instance types|classes other than X.metal
+  AWS_EC2_INSTANCE_TYPE: c6a.2xlarge
+  AWS_EC2_LAUNCH_TEMPLATE: lt-02e10a4f66261319d
+  AWS_EC2_LT_VERSION: 2
+  AWS_IAM_USERNAME: balena-tests-iam-User-1GXO3XP12N6LL
+  AWS_VPC_SECURITY_GROUP_IDS: sg-057937f4d89d9d51c
+  AWS_VPC_SUBNET_IDS: 'subnet-02d18a08ea4058574 subnet-0a026eae1df907a09'
+  # otherwise it tries to send data to an endpoint provided by a private project
+  # https://github.com/balena-io/analytics-backend
+  # .. which is not part of openBalena
+  BALENARC_NO_ANALYTICS: '1'  # https://github.com/balena-io/balena-cli/blob/master/lib/events.ts#L62-L70
+  DEBUG: '0'  # https://github.com/balena-io/balena-cli/issues/2447
+  RETRY: 3
+  SUBDOMAIN: auto
+
+jobs:
+  test:
+    runs-on: ["self-hosted", "X64", "distro:jammy"]  # tests require socat v1.7.4
+    timeout-minutes: 60
+    strategy:
+      fail-fast: true
+
+    steps:
+    - uses: actions/checkout@b80ff79f1755d06ba70441c368a6fe801f5f3a62
+      with:
+        # FIXME: remove once balenaBlocks/balenaVirt is a thing
+        submodules: true
+
+    - uses: aws-actions/configure-aws-credentials@43c891271eabca0b34d95f2453e2fe354928b16d
+      with:
+        aws-region: ${{ vars.AWS_REGION || 'us-east-1' }}
+        role-session-name: github-${{ github.job }}-${{ github.run_id }}-${{ github.run_attempt }}
+        # balena-io/environments-bases: aws/balenacloud/ephemeral-tests/balena-tests-iam.yml
+        role-to-assume: ${{ vars.AWS_IAM_ROLE }}
+
+    # https://github.com/pdcastro/ssh-uuid#why
+    # https://github.com/pdcastro/ssh-uuid#linux-debian-ubuntu-others
+    - name: install additional dependencies
+      shell: bash
+      run: |
+        set -ue
+        echo '::notice::install additional dependencies'
+
+        [[ '${{ vars.VERBOSE || 'false' }}' =~ on|On|Yes|yes|true|True ]] && set -x
+
+        source src/balena-tests/functions
+
+        mkdir -p "${RUNNER_TEMP}/ssh-uuid"
+
+        wget -q -O "${RUNNER_TEMP}/ssh-uuid/ssh-uuid" https://raw.githubusercontent.com/pdcastro/ssh-uuid/master/ssh-uuid.sh \
+          && chmod +x "${RUNNER_TEMP}/ssh-uuid/ssh-uuid" \
+          && ln -s "${RUNNER_TEMP}/ssh-uuid/ssh-uuid" "${RUNNER_TEMP}/ssh-uuid/scp-uuid"
+
+        with_backoff balena login --token '${{ secrets.BALENA_API_KEY }}'
+
+        balena version
+
+        "${RUNNER_TEMP}/ssh-uuid/scp-uuid" --help
+
+        grep -q "${RUNNER_TEMP}/ssh-uuid" "${GITHUB_PATH}" \
+          || echo "${RUNNER_TEMP}/ssh-uuid" >> "${GITHUB_PATH}"
+
+    - name: (pre)register test device
+      id: register-test-device
+      if: ${{ github.event_name == 'pull_request' && github.event.action != 'closed'}}
+      run: |
+        set -ue
+
+        [[ '${{ vars.VERBOSE || 'false' }}' =~ on|On|Yes|yes|true|True ]] && set -x
+
+        source src/balena-tests/functions
+
+        with_backoff balena login --token '${{ secrets.BALENA_API_KEY }}'
+
+        balena_device_uuid="$(openssl rand -hex 16)"
+
+        # https://www.balena.io/docs/learn/more/masterclasses/advanced-cli/#52-preregistering-a-device
+        balena device register '${{ inputs.fleet }}' --uuid "${balena_device_uuid}"
+
+        device_id="$(balena device "${balena_device_uuid}" | grep ^ID: | cut -c20-)"
+
+        # the actual version deployed depends on the AWS EC2/AMI, defined in AWS_EC2_LAUNCH_TEMPLATE
+        os_version="$(balena os versions ${{ vars.DEVICE_TYPE || 'generic-amd64' }} | head -n 1)"
+
+        balena config generate \
+          --version "${os_version}" \
+          --device "${balena_device_uuid}" \
+          --network ethernet \
+          --appUpdatePollInterval 10 \
+          $([[ '${{ vars.DEVELOPMENT_MODE || 'false' }}' =~ true ]] && echo '--dev') \
+          --output config.json
+
+        balena tag set balena ephemeral-test-device --device "${balena_device_uuid}"
+
+        github_vars=(GITHUB_ACTOR GITHUB_BASE_REF GITHUB_HEAD_REF GITHUB_JOB \
+          GITHUB_REF GITHUB_REF_NAME GITHUB_REF_TYPE GITHUB_REPOSITORY \
+          GITHUB_REPOSITORY_OWNER GITHUB_RUN_ATTEMPT GITHUB_RUN_ID GITHUB_RUN_NUMBER \
+          GITHUB_SHA GITHUB_WORKFLOW RUNNER_ARCH RUNNER_NAME RUNNER_OS)
+
+        for github_var in "${github_vars[@]}"; do
+            balena tag set ${github_var} "${!github_var}" --device "${balena_device_uuid}"
+        done
+
+        echo "balena_device_uuid=${balena_device_uuid}" >> "${GITHUB_OUTPUT}"
+        echo "balena_device_id=${device_id}" >> "${GITHUB_OUTPUT}"
+
+   # https://github.com/balena-io/balena-cli/issues/1543
+    - name: pin device to draft release
+      if: ${{ github.event_name == 'pull_request' && github.event.action != 'closed'}}
+      run: |
+        set -uae
+
+        [[ '${{ vars.VERBOSE || 'false' }}' =~ on|On|Yes|yes|true|True ]] && set -x
+
+        source src/balena-tests/functions
+
+        with_backoff balena login --token '${{ secrets.BALENA_API_KEY }}'
+
+        pr_id='${{ github.event.pull_request.id }}'
+        head_sha='${{ github.event.pull_request.head.sha || github.event.head_commit.id }}'
+        release_id="$(with_backoff balena releases '${{ inputs.fleet }}' --json \
+          | jq -r --arg pr_id "${pr_id}" --arg head_sha "${head_sha}" '.[]
+          | select(.release_tag[].tag_key=="balena-ci-commit-sha")
+          | select(.release_tag[].value==$head_sha)
+          | select(.release_tag[].tag_key=="balena-ci-id")
+          | select(.release_tag[].value==$pr_id).commit')"
+
+        with_backoff balena device pin \
+          ${{ steps.register-test-device.outputs.balena_device_uuid }} \
+          "${release_id}"
+
+        with_backoff balena device ${{ steps.register-test-device.outputs.balena_device_uuid }}
+
+    - name: configure test device environment
+      if: ${{ github.event_name == 'pull_request' && github.event.action != 'closed'}}
+      run: |
+        set -ue
+
+        [[ '${{ vars.VERBOSE || 'false' }}' =~ on|On|Yes|yes|true|True ]] && set -x
+
+        source src/balena-tests/functions
+
+        with_backoff balena login --token '${{ secrets.BALENA_API_KEY }}'
+
+        with_backoff balena env add VERBOSE "${{ vars.VERBOSE || 'false' }}" \
+          --device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
+
+        with_backoff balena env add BALENARC_NO_ANALYTICS '1' \
+          --device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
+
+        with_backoff balena env add DNS_TLD '${{ env.SUBDOMAIN }}.${{ inputs.dns_tld }}' \
+          --device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
+
+        with_backoff balena env add DB_HOST db \
+          --service api \
+          --device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
+
+        with_backoff balena env add REDIS_HOST redis:6379 \
+          --service api \
+          --device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
+
+
+        # to allow devices running locally to communicate to the local API, we can route
+        # to the local Docker network aliases instead of public DNS, since (a) DNS_TLD is
+        # guestfwd(ed) in QEMU to a special internal IP 10.0.2.100; (b) is proxied to
+        # haproxy network alias on device; and (c) made public with a wildcard DNS record
+        # (e.g.)
+        #
+        # $ dig +short api.auto.balena-devices.com
+        # 10.0.2.100
+        #
+
+        with_backoff balena env add API_HOST 'api.${{ env.SUBDOMAIN }}.${{ inputs.dns_tld }}' \
+          --device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
+
+        # not used but required for config.json to be valid
+        with_backoff balena env add DELTA_HOST 'delta.${{ env.SUBDOMAIN }}.${{ inputs.dns_tld }}' \
+          --device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
+
+        with_backoff balena env add REGISTRY2_HOST 'registry2.${{ env.SUBDOMAIN }}.${{ inputs.dns_tld }}' \
+          --device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
+
+        with_backoff balena env add VPN_HOST 'cloudlink.${{ env.SUBDOMAIN }}.${{ inputs.dns_tld }}' \
+          --device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
+
+        with_backoff balena env add HOST 'api.${{ env.SUBDOMAIN }}.${{ inputs.dns_tld }}' \
+          --service api \
+          --device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
+
+        with_backoff balena env add TOKEN_AUTH_CERT_ISSUER 'api.${{ env.SUBDOMAIN }}.${{ inputs.dns_tld }}' \
+          --service api \
+          --device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
+
+        with_backoff balena env add REGISTRY2_TOKEN_AUTH_ISSUER 'api.${{ env.SUBDOMAIN }}.${{ inputs.dns_tld }}' \
+          --service registry \
+          --device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
+
+        with_backoff balena env add REGISTRY2_TOKEN_AUTH_REALM 'https://api.${{ env.SUBDOMAIN }}.${{ inputs.dns_tld }}/auth/v1/token' \
+          --service registry \
+          --device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
+
+        with_backoff balena env add REGISTRY2_S3_REGION_ENDPOINT 's3.${{ env.SUBDOMAIN }}.${{ inputs.dns_tld }}' \
+          --device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
+
+        with_backoff balena env add WEBRESOURCES_S3_HOST 's3.${{ env.SUBDOMAIN }}.${{ inputs.dns_tld }}' \
+          --device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
+
+        # https://github.com/balena-io/cert-manager/blob/master/entry.sh#L255-L278
+        # cert-manager will restore the last wildcard certificate from AWS/S3 to avoid
+        # being rate limited by LetsEncrypt/ACME
+        with_backoff balena env add AWS_S3_BUCKET '${{ env.AWS_S3_CERTS_BUCKET }}' \
+          --service cert-manager \
+          --device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
+
+        # FIXME: still required?
+        with_backoff balena env add COMMON_REGION '${{ env.AWS_REGION }}' \
+          --device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
+
+        with_backoff balena env add SUPERUSER_EMAIL 'admin@${{ env.SUBDOMAIN }}.${{ inputs.dns_tld }}' \
+          --device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
+
+        with_backoff balena env add ORG_UNIT openBalena \
+          --device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
+
+
+        # unstable/unsupported functionality
+        with_backoff balena env add HIDE_UNVERSIONED_ENDPOINT 'false' \
+          --service api \
+          --device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
+
+        with_backoff balena env add RELEASE_ASSETS_TEST 'true' \
+          --service sut \
+          --device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
+
+    - name: configure test device secrets
+      if: ${{ github.event_name == 'pull_request' && github.event.action != 'closed'}}
+      run: |
+        set -ue
+
+        [[ '${{ vars.VERBOSE || 'false' }}' =~ on|On|Yes|yes|true|True ]] && set -x
+
+        source src/balena-tests/functions
+
+        with_backoff balena login --token '${{ secrets.BALENA_API_KEY }}'
+
+        # cert-manager requires it to get whoami information for the user
+        with_backoff balena env add API_TOKEN '${{ secrets.BALENA_API_KEY }}' \
+          --service cert-manager \
+          --device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
+
+        # cert-manager requires is to request wildcard SSL certificate from LetsEncrypt
+        with_backoff balena env add CLOUDFLARE_API_TOKEN '${{ secrets.CLOUDFLARE_API_TOKEN }}' \
+          --service cert-manager \
+          --device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
+
+        # AWS credentials to backup/restore PKI assets
+        with_backoff balena env add AWS_ACCESS_KEY_ID '${{ env.AWS_ACCESS_KEY_ID }}' \
+          --service cert-manager \
+          --device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
+
+        with_backoff balena env add AWS_SECRET_ACCESS_KEY '${{ env.AWS_SECRET_ACCESS_KEY }}' \
+          --service cert-manager \
+          --device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
+
+    - name: provision ephemeral test device
+      id: provision-test-device
+      if: ${{ github.event_name == 'pull_request' && github.event.action != 'closed'}}
+      run: |
+        set -ue
+
+        [[ '${{ vars.VERBOSE || 'false' }}' =~ on|On|Yes|yes|true|True ]] && set -x
+
+        for subnet_id in ${{ env.AWS_VPC_SUBNET_IDS }}; do
+            # spot, on-demand
+            for market_type in ${{ vars.MARKET_TYPES || 'spot' }}; do
+                # https://docs.aws.amazon.com/cli/latest/reference/ec2/run-instances.html
+                response="$(aws ec2 run-instances \
+                  --launch-template 'LaunchTemplateId=${{ env.AWS_EC2_LAUNCH_TEMPLATE }},Version=${{ env.AWS_EC2_LT_VERSION }}' \
+                  --instance-type '${{ env.AWS_EC2_INSTANCE_TYPE }}' \
+                  $([[ $market_type =~ spot ]] && echo '--instance-market-options MarketType=spot') \
+                  --security-group-ids '${{ env.AWS_VPC_SECURITY_GROUP_IDS }}' \
+                  --subnet-id "${subnet_id}" \
+                  --associate-public-ip-address \
+                  --user-data file://config.json \
+                  --tag-specifications "ResourceType=instance,Tags=[{Key=Name,Value=balena-tests},{Key=MarketType,Value=${market_type}},{Key=Owner,Value=${{ env.AWS_IAM_USERNAME }}},{Key=GITHUB_SHA,Value=${GITHUB_SHA}-tests}]" || true)"
+
+                [[ -n $response ]] && break
+            done
+            [[ -n $response ]] && break
+        done
+
+        [[ -z $response ]] && exit 1
+
+        instance_id="$(echo "${response}" | jq -r '.Instances[].InstanceId')"
+
+        aws ec2 wait instance-running --instance-ids "${instance_id}"
+
+        aws ec2 wait instance-status-ok --instance-ids "${instance_id}"
+
+        echo "instance_id=${instance_id}" >> "${GITHUB_OUTPUT}"
+
+      env:
+        AWS_ACCESS_KEY_ID: ${{ env.AWS_ACCESS_KEY_ID }}
+        AWS_SECRET_ACCESS_KEY: ${{ env.AWS_SECRET_ACCESS_KEY }}
+        AWS_DEFAULT_REGION: ${{ env.AWS_REGION }}
+
+    - name: provision SSH key
+      id: provision-ssh-key
+      # wait for cloud-config
+      # https://github.com/balena-os/cloud-config
+      timeout-minutes: 5
+      if: ${{ github.event_name == 'pull_request' && github.event.action != 'closed'}}
+      run: |
+        set -ue
+
+        [[ '${{ vars.VERBOSE || 'false' }}' =~ on|On|Yes|yes|true|True ]] && set -x
+
+        source src/balena-tests/functions
+
+        with_backoff balena login --token '${{ secrets.BALENA_API_KEY }}'
+
+        if ! [[ -e "${HOME}/.ssh/id_rsa" ]]; then
+            ssh-keygen -N '' \
+              -C "$(balena whoami | grep EMAIL | cut -c11-)" \
+              -f "${HOME}/.ssh/id_rsa"
+        fi
+
+        echo "::notice::check $(balena keys | wc -l) keys"
+
+        match=''
+        for key in $(balena keys | grep -v ID | awk '{print $1}'); do
+            fp=$(balena key ${key} | tail -n 1 | ssh-keygen -E md5 -lf /dev/stdin | awk '{print $2}')
+            if [[ $fp =~ $(ssh-keygen -E md5 -lf "${HOME}/.ssh/id_rsa" | awk '{print $2}') ]]; then
+                match="${key}"
+                break
+            fi
+        done
+
+        if [[ -z $match ]]; then
+            balena key add "${GITHUB_SHA}" "${HOME}/.ssh/id_rsa.pub"
+        else
+            balena keys
+        fi
+
+        pgrep ssh-agent || ssh-agent -a "${SSH_AUTH_SOCK}"
+
+        ssh-add "${HOME}/.ssh/id_rsa"
+
+        while ! [[ "$(ssh-uuid -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
+          ${{ steps.register-test-device.outputs.balena_device_uuid }}.balena \
+          cat /mnt/boot/config.json | jq -r .uuid)" =~ ${{ steps.register-test-device.outputs.balena_device_uuid }} ]]; do
+
+            echo "::warning::Still working..."
+            sleep "$(( (RANDOM % 5) + 5 ))s"
+        done
+
+        echo "key_id=${GITHUB_SHA}" >> "${GITHUB_OUTPUT}"
+
+      env:
+        SSH_AUTH_SOCK: /tmp/ssh_agent.sock
+
+    - name: wait for application
+      timeout-minutes: 10
+      if: ${{ github.event_name == 'pull_request' && github.event.action != 'closed'}}
+      run: |
+        set -ue
+
+        [[ '${{ vars.VERBOSE || 'false' }}' =~ on|On|Yes|yes|true|True ]] && set -x
+
+        source src/balena-tests/functions
+
+        with_backoff balena login --token '${{ secrets.BALENA_API_KEY }}'
+
+        balena whoami && ssh-add -l
+
+        while [[ "$(curl -X POST --silent --retry ${{ env.RETRY }} --fail \
+          'https://api.${{ inputs.environment }}/supervisor/v1/device' \
+          --header 'authorization: Bearer ${{ secrets.BALENA_API_KEY }}' \
+          --header 'Content-Type:application/json' \
+          --data '{"uuid": "${{ steps.register-test-device.outputs.balena_device_uuid }}", "method": "GET"}' \
+          --compressed | jq -r '.update_pending')" =~ ^true$ ]]; do
+
+            sleep "$(( ( RANDOM % ${{ env.RETRY }} ) + ${{ env.RETRY }} ))s"
+        done
+
+        # wait for services to start running
+        while with_backoff ssh-uuid -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
+          ${{ steps.register-test-device.outputs.balena_device_uuid }}.balena \
+          'balena ps -q | xargs balena inspect | jq -r .[].State.Status' \
+          | grep -E 'created|restarting|removing|paused|exited|dead'; do
+
+            echo "::warning::Still working..."
+            sleep "$(( (RANDOM % 30) + 30 ))s"
+        done
+
+        # wait for Docker healthchecks
+        while with_backoff ssh-uuid -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
+          ${{ steps.register-test-device.outputs.balena_device_uuid }}.balena \
+          'balena ps -q | xargs balena inspect \
+          | jq -r ".[] | select(.State.Health.Status!=null).Name + \":\" + .State.Health.Status"' \
+          | grep -E ':starting|:unhealthy'; do
+
+            echo "::warning::Still working..."
+            sleep "$(( (RANDOM % 30) + 30 ))s"
+        done
+
+      env:
+        SSH_AUTH_SOCK: /tmp/ssh_agent.sock
+
+    # (TBC) https://www.balena.io/docs/reference/supervisor/docker-compose/
+    # due to lack of long form depends_on support in compositions, restart to ensure all
+    # components are running with the latest configuration; preferred over restart via
+    # Supervisor API restart due to potential HTTP [timeouts](https://github.com/balena-os/balena-supervisor/issues/1157)
+    - name: restart components
+      timeout-minutes: 10
+      if: ${{ github.event_name == 'pull_request' && github.event.action != 'closed'}}
+      run: |
+        set -ue
+
+        [[ '${{ vars.VERBOSE || 'false' }}' =~ on|On|Yes|yes|true|True ]] && set -x
+
+        source src/balena-tests/functions
+
+        with_backoff balena login --token '${{ secrets.BALENA_API_KEY }}'
+
+        balena whoami && ssh-add -l
+
+        with_backoff ssh-uuid -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
+          ${{ steps.register-test-device.outputs.balena_device_uuid }}.balena \
+          "balena ps -aq | xargs balena inspect \
+          | jq -re '.[]
+          | select(.Name | contains(\"_supervisor\") | not).Id' \
+          | xargs balena restart"
+
+        # wait for Docker healthchecks
+        while with_backoff ssh-uuid -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
+          ${{ steps.register-test-device.outputs.balena_device_uuid }}.balena \
+          'balena ps -q | xargs balena inspect \
+          | jq -r ".[] | select(.State.Health.Status!=null).Name + \":\" + .State.Health.Status"' \
+          | grep -E ':starting|:unhealthy'; do
+
+            echo "::warning::Still working..."
+            sleep "$(( (RANDOM % 30) + 30 ))s"
+        done
+
+      env:
+        SSH_AUTH_SOCK: /tmp/ssh_agent.sock
+
+    - name: SUT&DUT
+      if: ${{ github.event_name == 'pull_request' && github.event.action != 'closed'}}
+      timeout-minutes: 20
+      # https://giters.com/gfx/example-github-actions-with-tty
+      # https://github.com/actions/runner/issues/241#issuecomment-924327172
+      shell: 'script -q -e -c "bash {0}"'
+      run: |
+        set -ue
+
+        [[ '${{ vars.VERBOSE || 'false' }}' =~ on|On|Yes|yes|true|True ]] && set -x
+
+        source src/balena-tests/functions
+
+        with_backoff balena login --token '${{ secrets.BALENA_API_KEY }}'
+
+        balena whoami && ssh-add -l
+
+        (with_backoff ssh-uuid -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
+          ${{ steps.register-test-device.outputs.balena_device_uuid }}.balena \
+          "balena ps -aq | xargs balena inspect \
+          | jq -re '.[] | select(.Name | contains(\"sut_\")).Id' \
+          | xargs balena logs -f") &
+
+        # tests service is working while its status == running
+        status=''
+        while [[ "$status" =~ Running ]]; do
+            status="$(curl --silent --retry ${{ env.RETRY }} --fail \
+              'https://api.${{ inputs.environment }}/supervisor/v2/applications/state' \
+              --header 'authorization: Bearer ${{ secrets.BALENA_API_KEY }}' \
+              --header 'Content-Type:application/json' \
+              --data '{"uuid": "${{ steps.register-test-device.outputs.balena_device_uuid }}", "method": "GET"}' \
+              --compressed | jq -r '.[].services.sut.status')"
+
+            echo "::warning::Still working..."
+            sleep "$(( ( RANDOM % ${{ env.RETRY }} ) + ${{ env.RETRY }} ))s"
+        done
+
+        # .. once the service exits with status == exited, it is assumed to be finished
+        status=''
+        while ! [[ "$status" =~ exited ]]; do
+            echo "::warning::Still working..."
+            status="$(curl --silent --retry ${{ env.RETRY }} --fail \
+              'https://api.${{ inputs.environment }}/supervisor/v2/applications/state' \
+              --header 'authorization: Bearer ${{ secrets.BALENA_API_KEY }}' \
+              --header 'Content-Type:application/json' \
+              --data '{"uuid": "${{ steps.register-test-device.outputs.balena_device_uuid }}", "method": "GET"}' \
+              --compressed | jq -r '.[].services.sut.status')"
+
+            sleep "$(( ( RANDOM % ${{ env.RETRY }} ) + ${{ env.RETRY }} ))s"
+        done
+
+        # .. check its exit code
+        expected_exit_code=0
+        actual_exit_code="$(with_backoff ssh-uuid -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
+          ${{ steps.register-test-device.outputs.balena_device_uuid }}.balena \
+          "balena ps -aq | xargs balena inspect \
+          | jq -re '.[] | select(.Name | contains(\"sut_\")).State.ExitCode'")"
+
+        [[ $expected_exit_code -eq $actual_exit_code ]] || false
+
+      env:
+        SSH_AUTH_SOCK: /tmp/ssh_agent.sock
+        ATTEMPTS: 2
+
+    - name: remove SSH key
+      if: always()
+      continue-on-error: true
+      run: |
+        set -ue
+
+        [[ '${{ vars.VERBOSE || 'false' }}' =~ on|On|Yes|yes|true|True ]] && set -x
+
+        source src/balena-tests/functions
+
+        with_backoff balena login --token '${{ secrets.BALENA_API_KEY }}'
+
+        with_backoff balena keys | grep ${{ steps.provision-ssh-key.outputs.key_id }} \
+          | awk '{print $1}' | xargs balena key rm --yes
+
+        pgrep ssh-agent && (pgrep ssh-agent | xargs kill)
+
+        rm -f /tmp/ssh_agent.sock
+
+    - name: destroy balena test device
+      if: always()
+      continue-on-error: true
+      run: |
+        set -ue
+
+        [[ '${{ vars.VERBOSE || 'false' }}' =~ on|On|Yes|yes|true|True ]] && set -x
+
+        source src/balena-tests/functions
+
+        with_backoff balena login --token '${{ secrets.BALENA_API_KEY }}'
+
+        with_backoff balena device rm ${{ steps.register-test-device.outputs.balena_device_uuid }} --yes
+
+      env:
+        AWS_ACCESS_KEY_ID: ${{ env.AWS_ACCESS_KEY_ID }}
+        AWS_SECRET_ACCESS_KEY: ${{ env.AWS_SECRET_ACCESS_KEY }}
+        AWS_DEFAULT_REGION: ${{ env.AWS_REGION }}
+
+    # always destroy test EC2 instances even if the workflow is cancelled
+    - name: destroy AWS test device
+      if: always()
+      run: |
+        set -ue
+
+        [[ '${{ vars.VERBOSE || 'false' }}' =~ on|On|Yes|yes|true|True ]] && set -x
+
+        source src/balena-tests/functions
+
+        if [[ -n '${{ steps.provision-test-device.outputs.instance_id }}' ]]; then
+            with_backoff aws ec2 terminate-instances \
+              --instance-ids ${{ steps.provision-test-device.outputs.instance_id }}
+        fi
+
+        with_backoff aws ec2 describe-instances --filters Name=tag:GITHUB_SHA,Values=${GITHUB_SHA}-tests \
+          | jq -r .Reservations[].Instances[].InstanceId \
+          | xargs aws ec2 terminate-instances --instance-ids
+
+        stale_instances=$(mktemp)
+        aws ec2 describe-instances --filters \
+          Name=tag:Name,Values=balena-tests \
+          Name=instance-state-name,Values=running \
+          | jq -re '.Reservations[].Instances[].InstanceId + " " + .Reservations[].Instances[].LaunchTime' > ${stale_instances} || true
+
+        if test -s "${stale_instances}"; then
+            while IFS= read -r line; do
+                instance_id=$(echo ${line} | awk '{print $1}')
+                launch_time=$(echo ${line} | awk '{print $2}')
+                now=$(date +%s)
+                then=$(date --date ${launch_time} +%s)
+                days_since_launch=$(( (now - then) / 86400 ))
+                if [[ -n $days_since_launch ]] && [[ $days_since_launch -ge 1 ]]; then
+                    with_backoff aws ec2 terminate-instances --instance-ids ${instance_id}
+                fi
+            done <${stale_instances}
+            rm -f ${stale_instances}
+        fi
+
+      env:
+        AWS_ACCESS_KEY_ID: ${{ env.AWS_ACCESS_KEY_ID }}
+        AWS_SECRET_ACCESS_KEY: ${{ env.AWS_SECRET_ACCESS_KEY }}
+        AWS_DEFAULT_REGION: ${{ env.AWS_REGION }}
+
+    # remove orphaned ACME DNS-01 validation records
+    # https://letsencrypt.org/docs/challenge-types/#dns-01-challenge
+    # FIXME: clean up older _acme-challenge.auto TXT records
+    - name: cleanup-dns-records
+      if: always()
+      continue-on-error: true
+      run: |
+        set -ue
+
+        [[ '${{ vars.VERBOSE || 'false' }}' =~ on|On|Yes|yes|true|True ]] && set -x
+
+        if [[ -n '${{ steps.register-test-device.outputs.balena_device_uuid }}' ]]; then
+            match="${{ steps.register-test-device.outputs.balena_device_uuid }}.${{ env.SUBDOMAIN }}"
+
+            zone_id="$(curl --silent --retry ${{ env.RETRY }} \
+              "https://api.cloudflare.com/client/v4/zones?name=${{ inputs.dns_tld }}" \
+              -H 'Authorization: Bearer ${{ secrets.CLOUDFLARE_API_TOKEN }}' | jq -r '.result[].id')"
+
+            for record in "$(curl --silent --retry ${{ env.RETRY }} \
+              "https://api.cloudflare.com/client/v4/zones/${zone_id}/dns_records" \
+              -H 'Authorization: Bearer ${{ secrets.CLOUDFLARE_API_TOKEN }}' \
+              | jq -r --arg match "${match}" '.result[] | select(((.type=="TXT") and (.name | contains($match))))' \
+              | base64)"; do
+
+                json="$(echo "${record}" | base64 -d | jq -r)"
+                id="$(echo "${json}" | jq -r .id)"
+                name="$(echo "${json}" | jq -r .name)"
+
+                if [[ -n $id ]] && [[ -n $name ]]; then
+                    echo "::warning::Orphaned DNS record ${name} (${id})..."
+
+                    if [[ -z $DRY_RUN ]]; then
+                        curl -X DELETE --silent --retry ${{ env.RETRY }} \
+                          "https://api.cloudflare.com/client/v4/zones/${zone_id}/dns_records/${id}" \
+                          -H 'Authorization: Bearer ${{ secrets.CLOUDFLARE_API_TOKEN }}'
+                    fi
+                fi
+            done
+        fi
+
+      env:
+        DRY_RUN: true

.gitignore

@@ -1,6 +1,3 @@
 .DS_Store
-.project
-.vagrant/
-config/
-src/
-package-lock.json
+.balena
+**/.env

.openbalenarc

@@ -1,50 +0,0 @@
-#!/bin/bash
-
-alias dc="/home/vagrant/openbalena/scripts/compose"
-
-function enter () {
-	if [[ $# -lt 1 ]]; then			
-		echo "Usage: enter <service name> [command]"
-		echo "  "
-		echo "  Runs a [command] in the service specified."
-		echo "  "
-		echo "  command:"
-		echo "    (default) /bin/bash"
-		echo "  "
-		echo "  example:"
-		echo "    enter api          # this will run the command '/bin/bash' in the API service, providing a shell prompt"
-		echo "    enter api uptime   # this will run the command 'uptime' in the API service, and return"
-		return 1
-	fi
-
-
-	service="$1"
-	shift
-	COMMAND=/bin/bash
-	if [[ $# -gt 0 ]]; then
-		COMMAND="$@"
-	fi
-	dc exec ${service} /bin/bash -c "${COMMAND}"
-}
-
-function logs () {
-	if [[ $# -lt 1 ]]; then			
-		echo "Usage: logs <service name> [options]"
-		echo "  "
-		echo "  Shows the logs from journalctl in the service specified."
-		echo "  "
-		echo "  options:"
-		echo "    -f  tail the log stream"
-		echo "    -n  number of lines to take"
-		echo "  "
-		echo "  example:"
-		echo "    logs api -fn100 # this will tail the API log, starting with the last 100 lines"
-		return 1
-	fi
-
-	service="$1"
-	shift
-	enter ${service} journalctl "$@"
-}
-
-cd /home/vagrant/openbalena

Makefile

@@ -1,4 +1,135 @@
-.PHONY: lint
+SHELL := bash
 
-lint:
-	shellcheck scripts/*
+# export all variables to child processes by default
+export
+
+# Include the .env file
+include .env
+
+DNS_TLD ?= $(error DNS_TLD not set)
+TMPKI := $(shell mktemp)
+STAGING_PKI ?= /usr/local/share/ca-certificates
+PRODUCTION_MODE ?= true
+ORG_UNIT ?= openBalena
+SUPERUSER_EMAIL ?= admin@$(DNS_TLD)
+
+.NOTPARALLEL: $(DOCKERCOMPOSE)
+
+.PHONY: help
+help: ## Print help message
+	@echo -e "$$(grep -hE '^\S+:.*##' $(MAKEFILE_LIST) | sed -e 's/:.*##\s*/:/' -e 's/^\(.\+\):\(.*\)/\\x1b[36m\1\\x1b[m:\2/' | column -c2 -t -s :)"
+
+.PHONY: lint
+lint: ## Lint shell scripts with shellcheck
+	find . -type f -name *.sh | xargs shellcheck
+
+.PHONY: verify
+verify: ## Ping the public API endpoint
+	curl --fail --retry 3 https://api.$(DNS_TLD)/ping
+	@printf '\n'
+
+# Write all supported variables to .env, whether they have been provided or not.
+# If they already exist in the .env they will be retained.
+# The existing .env takes priority over envs provided from the command line.
+.PHONY: config
+config: ## Rewrite the .env config from current context (env vars + env args + existing .env)
+ifneq ($(CLOUDFLARE_API_TOKEN),)
+ifneq ($(GANDI_API_TOKEN),)
+	$(error "CLOUDFLARE_API_TOKEN and GANDI_API_TOKEN cannot both be set")
+endif
+endif
+	@rm -f .env
+	@echo "DNS_TLD=$(DNS_TLD)" >> .env
+	@echo "ORG_UNIT=$(ORG_UNIT)" >> .env
+	@echo "SUPERUSER_EMAIL=$(SUPERUSER_EMAIL)" >> .env
+	@echo "PRODUCTION_MODE=$(PRODUCTION_MODE)" >> .env
+	@echo "GANDI_API_TOKEN=$(GANDI_API_TOKEN)" >> .env
+	@echo "CLOUDFLARE_API_TOKEN=$(CLOUDFLARE_API_TOKEN)" >> .env
+	@echo "ACME_EMAIL=$(ACME_EMAIL)" >> .env
+	@echo "HAPROXY_CRT=$(HAPROXY_CRT)" >> .env
+	@echo "HAPROXY_KEY=$(HAPROXY_KEY)" >> .env
+	@echo "ROOT_CA=$(ROOT_CA)" >> .env
+	@$(MAKE) showenv
+
+.PHONY: up
+up: config ## Start all services
+	@docker compose up --build -d
+	@until [[ $$(docker compose ps api --format json | jq -r '.Health') =~ healthy ]]; do printf '.'; sleep 3; done
+	@printf '\n'
+	@$(MAKE) showenv
+	@$(MAKE) showpass
+
+.PHONY: showenv
+showenv: ## Print the current contents of the .env config
+	@cat <.env
+	@printf '\n'
+
+.PHONY: printenv
+printenv: ## Print the current environment variables
+	@printenv
+
+.PHONY: showpass
+showpass: ## Print the superuser password
+	@docker compose exec api cat config/env | grep SUPERUSER_PASSWORD
+	@printf '\n'
+
+.PHONY: down
+down: ## Stop all services
+	@docker compose stop
+
+.PHONY: stop
+stop: down ## Alias for 'make down'
+
+.PHONY: restart
+restart: ## Restart all services
+	@docker compose restart
+
+.PHONY: update
+update: # Pull and deploy latest changes from git
+	@git pull
+	@$(MAKE) up
+
+.PHONY: destroy ## Stop and remove any existing containers and volumes
+destroy:
+	@docker compose down --volumes --remove-orphans
+
+.PHONY: clean
+clean: destroy ## Alias for 'make destroy'
+
+.PHONY: self-signed
+self-signed: ## Install self-signed CA certificates
+	@sudo mkdir -p .balena $(STAGING_PKI)
+
+	@true | openssl s_client -showcerts -connect api.$(DNS_TLD):443 \
+	  | awk '/BEGIN CERTIFICATE/,/END CERTIFICATE/ {print $0}' > $(TMPKI).ca
+
+	@cat <$(TMPKI).ca | openssl x509 -text \
+	  | awk '/BEGIN CERTIFICATE/,/END CERTIFICATE/ {print $0}' > $(TMPKI).srv
+
+	@diff --suppress-common-lines --unchanged-line-format= \
+	  $(TMPKI).srv \
+	  $(TMPKI).ca | sudo tee $(STAGING_PKI)/ca-$(DNS_TLD).crt || true
+
+	@sudo update-ca-certificates
+	@cat <$(STAGING_PKI)/ca-$(DNS_TLD).crt | sudo tee .balena/ca-$(DNS_TLD).pem
+
+# FIXME: refactor this function to use 'make up'
+.PHONY: auto-pki
+auto-pki: config # Start all services using LetsEncrypt and ACME
+	@docker compose exec cert-manager rm -f /certs/export/chain.pem
+	@docker compose up -d
+	@until docker compose logs cert-manager | grep -Eq "/certs/export/chain.pem Certificate will not expire in [0-9] days"; do printf '.'; sleep 3; done
+	@until docker compose logs cert-manager | grep -q "subject=CN = ${DNS_TLD}"; do printf '.'; sleep 3; done
+	@until docker compose logs cert-manager | grep -q "issuer=C = US, O = Let's Encrypt, CN = R3"; do printf '.'; sleep 3; done
+	@until [[ $$(docker compose ps haproxy --format json | jq -r '.Health') =~ healthy ]]; do printf '.'; sleep 3; done
+	@printf '\n'
+	@$(MAKE) showenv
+	@$(MAKE) showpass
+
+.PHONY: pki-custom
+pki-custom: up ## Alias for 'make up'
+
+.PHONY: deploy
+deploy: up ## Alias for 'make up'
+
+.DEFAULT_GOAL = help

README.md

@@ -1,6 +1,8 @@
-<img alt="openBalena" src="docs/assets/openbalena-logo.svg" height="82">
+[![Flowzone](https://github.com/balena-io/open-balena/actions/workflows/flowzone.yml/badge.svg)](https://github.com/balena-io/open-balena/actions/workflows/flowzone.yml)
 
----
+![](./docs/images/openbalena-logo.svg)
+
+[![deploy button](https://balena.io/deploy.svg)](https://dashboard.balena-cloud.com/deploy?repoUrl=https://github.com/balena-io/open-balena)
 
 OpenBalena is a platform to deploy and manage connected devices. Devices run
 [balenaOS][balena-os-website], a host operating system designed for running
@@ -25,42 +27,29 @@ To learn more about openBalena, visit [balena.io/open][open-balena-website].
 - **Built-in VPN**: Access your devices regardless of their network environment
 
 
-## Roadmap
-
-OpenBalena is currently in beta. While fully functional, it lacks features we
-consider important before we can comfortably call it production-ready. During
-this phase, don’t be alarmed if things don’t work as expected just yet (and
-please let us know about any bugs or errors you encounter!). The following
-improvements and new functionality is planned:
-
-- Full documentation
-- Full test suite
-- Simplified deployment
-- Remote host OS updates
-- Support for custom device types
-
-
-## Contributing
-
-Everyone is welcome to contribute to openBalena. There are many different ways
-to get involved apart from submitting pull requests, including helping other
-users on the [forums][forums], reporting or triaging [issues][issue-tracker],
-reviewing and discussing [pull requests][pulls], or just spreading the word.
-
-All of openBalena is hosted on GitHub. Apart from its constituent components,
-which are the [API][open-balena-api], [VPN][open-balena-vpn], [Registry][open-balena-registry],
-[S3 storage service][open-balena-s3], and [Database][open-balena-db], contributions
-are also welcome to its client-side software such as the [balena CLI][balena-cli],
-the [balena SDK][balena-sdk], [balenaOS][balena-os] and [balenaEngine][balena-engine].
-
-
 ## Getting Started
 
-Our [Getting Started][getting-started] guide is the most direct path to getting
+Our [Getting Started guide][getting-started] is the most direct path to getting
 an openBalena installation up and running and successfully deploying your
 application to your device(s).
 
 
+## Compatibility
+
+The current release of openBalena has the following minimum version requirements:
+
+- balenaOS v5.2.8
+- balena CLI v18.2.2
+
+If you are updating from previous openBalena versions, ensure you update the balena
+CLI and re-provision any devices to at least the minimum required versions in order
+for them to be fully compatible with this release, as some features may not work.
+
+While in-place openBalena upgrades may succeed, when performing major updates, it is
+recommended for a new instance to be deployed in parallel with the existing one, followed
+by copying state across and pointing a test device to the new instance.
+
+
 ## Documentation
 
 While we're still working on the project documentation, please refer to the
@@ -89,9 +78,97 @@ for help, or contribute by answering questions posted by fellow openBalena users
 Please do not use the issue tracker for support-related questions.
 
 
+## Contributing
+
+Everyone is welcome to contribute to openBalena. There are many different ways
+to get involved apart from submitting pull requests, including helping other
+users on the [forums][forums], reporting or triaging [issues][issue-tracker],
+reviewing and discussing [pull requests][pulls], or just spreading the word.
+
+All of openBalena is hosted on GitHub. Apart from its constituent components,
+which are the [API][open-balena-api], [VPN][open-balena-vpn], [Registry][open-balena-registry],
+[S3 storage service][open-balena-s3], and [Database][open-balena-db], contributions
+are also welcome to its client-side software such as the [balena CLI][balena-cli],
+the [balena SDK][balena-sdk], [balenaOS][balena-os] and [balenaEngine][balena-engine].
+
+
+## Roadmap
+
+OpenBalena is currently in beta. While fully functional, it lacks features we
+consider important before we can comfortably call it production-ready. During
+this phase, don’t be alarmed if things don’t work as expected just yet (and
+please let us know about any bugs or errors you encounter!). The following
+improvements and new functionality is planned:
+
+- Full documentation
+- Full test suite
+- Simplified deployment
+- Remote host OS updates
+- Support for custom device types
+
+
+## Differences between openBalena and balenaCloud
+
+Whilst openBalena and balenaCloud share the same core technology, there are some key
+differences. First, openBalena is self-hosted, whereas balenaCloud is hosted by balena and
+therefore handles security, maintenance, scaling, and reliability of all the backend
+services. OpenBalena is also single user, whereas balenaCloud supports multiple users and
+organizations. OpenBalena also lacks some of the commercial features that define
+balenaCloud, such as the web-based dashboard and updates with binary container deltas.
+
+The following table contains the main differences between both:
+
+| openBalena                                                                                 | balenaCloud                                                                                                                                                                                               |
+| ------------------------------------------------------------------------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| Device updates using full Docker images                                                    | Device updates using [delta images](https://www.balena.io/docs/learn/deploy/delta/)                                                                                                                       |
+| Support for a single user                                                                  | Support for [multiple users](https://www.balena.io/docs/learn/manage/account/#application-members)                                                                                                        |
+| Self-hosted deployment and scaling                                                         | balena-managed scaling and deployment                                                                                                                                                                     |
+| Community support via [forums][forums]                                                     | Private support on [paid plans](https://www.balena.io/pricing/)                                                                                                                                           |
+| Build locally and deploy via `balena-cli`                                                  | Build remotely with native builders using [`balena push`](https://www.balena.io/docs/learn/deploy/deployment/#balena-push) or  [`git push`](https://www.balena.io/docs/learn/deploy/deployment/#git-push) |
+| No public device URL support                                                               | Serve websites directly from device with [public device URLs](https://www.balena.io/docs/learn/manage/actions/#enable-public-device-url)                                                                  |
+| Management via `balena-cli` only                                                           | Cloud-based device management dashboard                                                                                                                                                                   |
+| Download images from [balena.io][balena-os-website] and configure locally via `balena-cli` | Download configured images directly from the dashboard                                                                                                                                                    |
+| No remote device diagnostics                                                               | Remote device diagnostics                                                                                                                                                                                 |
+
+Additionally, refer back to the [roadmap](#roadmap) above for planned but not yet
+implemented features.
+
+
 ## License
 
-OpenBalena is licensed under the terms of AGPL v3. See [LICENSE](LICENSE) for details.
+OpenBalena is licensed under the terms of AGPL v3. See [LICENSE] for details.
+
+
+## FAQ
+
+### How do you ensure continuity of openBalena? Are there security patches on openBalena?
+openBalena is an open source initiative which is mostly driven by us, but it also gets
+contributions from the community. We work to keep openBalena as up to date as our
+bandwidth allows, especially with security patches. That said, we do not have a policy or
+guarantee of a software release schedule. However, it is in our best interest to keep
+openBalena updated and patched since we also use it for balenaCloud.
+
+### How do you ensure the "Join" command actually works between openBalena and
+balenaCloud?
+The `balena join ..` command is frequently used for moving devices between openBalena,
+and balenaCloud environments. This command extends `balena os configure ..`, which is the
+basic tool balena uses for configuring devices.
+
+### Is it "production ready"?
+While we actually have some rather large fleets using openBalena, we consider it to be
+perpetually in "beta". This means potentially introducing breaking changes between
+releases.
+
+### Can new device type be added to openBalena?
+openBalena imports the following public [device-types] "out of the box". You can specify
+your own contracts repository by overriding `CONTRACTS_PUBLIC_REPO_NAME`,
+`CONTRACTS_PUBLIC_REPO_OWNER` and `IMAGE_STORAGE_BUCKET` environment variables on the API
+service/container.
+
+### Are there open-source UI dashboards from the community for openBalena?
+Yes! Here are a few:
+- [open-balena-admin / open-balena-ui](https://github.com/dcaputo-harmoni/open-balena-admin) by [dcaputo-harmoni](https://github.com/dcaputo-harmoni) who first posted about [here](https://forums.balena.io/t/open-balena-admin-an-admin-interface-for-openbalena/355324) in our Forums :)
+- [open-balena-dashboard](https://github.com/Razikus/open-balena-dashboard) by [Razikus](https://github.com/Razikus)
 
 
 [balena-cli]: https://github.com/balena-io/balena-cli
@@ -104,10 +181,14 @@ OpenBalena is licensed under the terms of AGPL v3. See [LICENSE](LICENSE) for de
 [forums]: https://forums.balena.io/c/open-balena
 [getting-started]: https://balena.io/open/docs/getting-started
 [issue-tracker]: https://github.com/balena-io/open-balena/issues
+[LICENSE]: https://github.com/balena-io/open-balena/blob/master/LICENSE
+[open-balena-admin / open-balena-ui]: https://github.com/dcaputo-harmoni/open-balena-admin
 [open-balena-api]: https://github.com/balena-io/open-balena-api
+[open-balena-dashboard]: https://github.com/Razikus/open-balena-dashboard
 [open-balena-db]: https://github.com/balena-io/open-balena-db
 [open-balena-registry]: https://github.com/balena-io/open-balena-registry
 [open-balena-s3]: https://github.com/balena-io/open-balena-s3
 [open-balena-vpn]: https://github.com/balena-io/open-balena-vpn
 [open-balena-website]: https://balena.io/open
 [pulls]: https://github.com/balena-io/open-balena/pulls
+[device-types]: https://github.com/balena-io/contracts/blob/master/contracts/hw.device-type

VERSION

@@ -1 +1 @@
-0.2.0
+4.0.4

Vagrantfile

@@ -1,36 +0,0 @@
-Vagrant.require_version '>= 2.0.0'
-
-[ 'vagrant-vbguest', 'vagrant-docker-compose' ].each do |p|
-  unless Vagrant.has_plugin?(p)
-    raise "Please install missing plugin: vagrant plugin install #{p}"
-  end
-end
-
-Vagrant.configure('2') do |config|
-  config.vm.define 'openbalena'
-  config.vm.hostname = 'openbalena-vagrant'
-  config.vm.box = 'bento/ubuntu-18.04'
-
-  config.vm.network "public_network",
-    use_dhcp_assigned_default_route: true
-
-  config.vm.synced_folder '.', '/vagrant', disabled: true
-  config.vm.synced_folder '.', '/home/vagrant/openbalena'
-
-  config.ssh.forward_agent = true
-
-  config.vm.provision :docker
-  config.vm.provision :docker_compose
-
-  $provision = <<-SCRIPT
-    touch /home/vagrant/.bashrc
-    grep -Fxq 'source /home/vagrant/openbalena/.openbalenarc' /home/vagrant/.bashrc || echo 'source /home/vagrant/openbalena/.openbalenarc' >> /home/vagrant/.bashrc
-
-    curl -o- https://raw.githubusercontent.com/creationix/nvm/v0.34.0/install.sh | bash
-    source "/home/vagrant/.nvm/nvm.sh" # This loads nvm
-    nvm install 10.15.0 && nvm use 10.15.0
-  SCRIPT
-
-  config.vm.provision :shell, privileged: false, inline: $provision
-
-end

balena.yml

@@ -0,0 +1,26 @@
+name: openBalena
+type: sw.application
+description: https://www.balena.io/open
+post-provisioning: |
+  [![Flowzone](https://github.com/balena-io/open-balena/actions/workflows/flowzone.yml/badge.svg)](https://github.com/balena-io/open-balena/actions/workflows/flowzone.yml)
+
+  ## Getting Started
+
+  * https://open-balena.pages.dev/#getting-started
+
+assets:
+  repository:
+    type: blob.asset
+    data:
+      url: 'https://github.com/balena-io/open-balena'
+  logo:
+    type: blob.asset
+    data:
+      url: 'https://raw.githubusercontent.com/balena-io/open-balena/master/logo.png'
+data:
+  defaultDeviceType: generic-amd64
+  supportedDeviceTypes:
+    - generic-amd64
+    - genericx86-64-ext
+    - intel-nuc
+version: 4.0.4

compose/common.yml

@@ -1,17 +0,0 @@
-version: '2.1'
-
-services:
-  component:
-    cap_add:
-      - SYS_ADMIN
-      - SYS_RESOURCE
-    environment:
-      - CONFD_BACKEND=ENV
-    tmpfs:
-      - /run
-      - /sys/fs/cgroup
-    privileged: true
-
-  system:
-    security_opt:
-      - seccomp:unconfined

compose/services.yml

@@ -1,164 +0,0 @@
-version: '2.1'
-
-volumes:
-  db:
-  registry:
-  s3:
-  redis:
-
-services:
-  api:
-    extends:
-      file: ./common.yml
-      service: component
-    image: balena/open-balena-api:${OPENBALENA_API_VERSION_TAG:-master}
-    depends_on:
-      - db
-      - s3
-      - redis
-    environment:
-      API_VPN_SERVICE_API_KEY: ${OPENBALENA_API_VPN_SERVICE_API_KEY}
-      BALENA_ROOT_CA: ${OPENBALENA_ROOT_CA}
-      COOKIE_SESSION_SECRET: ${OPENBALENA_COOKIE_SESSION_SECRET}
-      DB_HOST: db.${OPENBALENA_HOST_NAME}
-      DB_PASSWORD: docker
-      DB_PORT: 5432
-      DB_USER: docker
-      DELTA_HOST: delta.${OPENBALENA_HOST_NAME}
-      DEVICE_CONFIG_OPENVPN_CONFIG: ${OPENBALENA_VPN_CONFIG}
-      DEVICE_CONFIG_OPENVPN_CA: ${OPENBALENA_VPN_CA_CHAIN}
-      DEVICE_CONFIG_SSH_AUTHORIZED_KEYS: ${OPENBALENA_SSH_AUTHORIZED_KEYS}
-      HOST: api.${OPENBALENA_HOST_NAME}
-      IMAGE_MAKER_URL: img.${OPENBALENA_HOST_NAME}
-      IMAGE_STORAGE_BUCKET: resin-production-img-cloudformation
-      IMAGE_STORAGE_PREFIX: resinos
-      IMAGE_STORAGE_ENDPOINT: s3.amazonaws.com
-      JSON_WEB_TOKEN_EXPIRY_MINUTES: 10080
-      JSON_WEB_TOKEN_SECRET: ${OPENBALENA_JWT_SECRET}
-      MIXPANEL_TOKEN: __unused__
-      PRODUCTION_MODE: '${OPENBALENA_PRODUCTION_MODE}'
-      PUBNUB_PUBLISH_KEY: __unused__
-      PUBNUB_SUBSCRIBE_KEY: __unused__
-      REDIS_HOST: redis.${OPENBALENA_HOST_NAME}
-      REDIS_PORT: 6379
-      REGISTRY2_HOST: registry.${OPENBALENA_HOST_NAME}
-      REGISTRY_HOST: registry.${OPENBALENA_HOST_NAME}
-      SENTRY_DSN:
-      TOKEN_AUTH_BUILDER_TOKEN: ${OPENBALENA_TOKEN_AUTH_BUILDER_TOKEN}
-      TOKEN_AUTH_CERT_ISSUER: api.${OPENBALENA_HOST_NAME}
-      TOKEN_AUTH_CERT_KEY: ${OPENBALENA_TOKEN_AUTH_KEY}
-      TOKEN_AUTH_CERT_KID: ${OPENBALENA_TOKEN_AUTH_KID}
-      TOKEN_AUTH_CERT_PUB: ${OPENBALENA_TOKEN_AUTH_PUB}
-      TOKEN_AUTH_JWT_ALGO: 'ES256'
-      VPN_HOST: vpn.${OPENBALENA_HOST_NAME}
-      VPN_PORT: 443
-      VPN_SERVICE_API_KEY: ${OPENBALENA_VPN_SERVICE_API_KEY}
-      SUPERUSER_EMAIL: ${OPENBALENA_SUPERUSER_EMAIL}
-      SUPERUSER_PASSWORD: ${OPENBALENA_SUPERUSER_PASSWORD}
-
-  registry:
-    extends:
-      file: ./common.yml
-      service: component
-    image: balena/open-balena-registry:${OPENBALENA_REGISTRY_VERSION_TAG:-master}
-    depends_on:
-      - api
-      - s3
-      - redis
-    volumes:
-      - registry:/data
-    environment:
-      API_TOKENAUTH_CRT: ${OPENBALENA_TOKEN_AUTH_PUB}
-      BALENA_REGISTRY2_HOST: registry.${OPENBALENA_HOST_NAME}
-      BALENA_ROOT_CA: ${OPENBALENA_ROOT_CA}
-      BALENA_TOKEN_AUTH_ISSUER: api.${OPENBALENA_HOST_NAME}
-      BALENA_TOKEN_AUTH_REALM: https://api.${OPENBALENA_HOST_NAME}/auth/v1/token
-      COMMON_REGION: 
-      REGISTRY2_S3_BUCKET: 
-      REGISTRY2_S3_KEY: 
-      REGISTRY2_S3_SECRET: 
-      REGISTRY2_SECRETKEY: ${OPENBALENA_REGISTRY_SECRET_KEY}
-      REGISTRY2_STORAGEPATH: /data
-
-  vpn:
-    extends:
-      file: ./common.yml
-      service: component
-    image: balena/open-balena-vpn:${OPENBALENA_VPN_VERSION_TAG:-master}
-    depends_on:
-      - api
-    cap_add:
-      - NET_ADMIN
-    environment:
-      API_SERVICE_API_KEY: ${OPENBALENA_API_VPN_SERVICE_API_KEY}
-      BALENA_API_HOST: api.${OPENBALENA_HOST_NAME}
-      BALENA_ROOT_CA: ${OPENBALENA_ROOT_CA}
-      BALENA_VPN_PORT: 443
-      PRODUCTION_MODE: '${OPENBALENA_PRODUCTION_MODE}'
-      RESIN_VPN_GATEWAY: 10.2.0.1
-      SENTRY_DSN: 
-      VPN_HAPROXY_USEPROXYPROTOCOL: 'true'
-      VPN_OPENVPN_CA_CRT: ${OPENBALENA_VPN_CA}
-      VPN_OPENVPN_SERVER_CRT: ${OPENBALENA_VPN_SERVER_CRT}
-      VPN_OPENVPN_SERVER_DH: ${OPENBALENA_VPN_SERVER_DH}
-      VPN_OPENVPN_SERVER_KEY: ${OPENBALENA_VPN_SERVER_KEY}
-      VPN_SERVICE_API_KEY: ${OPENBALENA_VPN_SERVICE_API_KEY}
-
-  db:
-    extends:
-      file: ./common.yml
-      service: system
-    image: balena/open-balena-db:${OPENBALENA_DB_VERSION_TAG:-master}
-    volumes:
-      - db:/var/lib/postgresql/data
-
-  s3:
-    extends:
-      file: ./common.yml
-      service: system
-    image: balena/open-balena-s3:${OPENBALENA_S3_VERSION_TAG:-master}
-    volumes:
-      - s3:/export
-
-  redis:
-    extends:
-      file: ./common.yml
-      service: system
-    image: redis:alpine
-    volumes:
-      - redis:/data
-
-  haproxy:
-    extends:
-      file: ./common.yml
-      service: system
-    build: ../haproxy
-    depends_on:
-      - api
-      - registry
-      - vpn
-      - db
-      - s3
-      - redis
-    ports:
-      - "80:80"
-      - "443:443"
-      - "3128:3128"
-    expose:
-      - "222"
-      - "5432"
-      - "6379"
-    networks:
-      default:
-        aliases:
-          - api.${OPENBALENA_HOST_NAME}
-          - registry.${OPENBALENA_HOST_NAME}
-          - vpn.${OPENBALENA_HOST_NAME}
-          - db.${OPENBALENA_HOST_NAME}
-          - s3.${OPENBALENA_HOST_NAME}
-          - redis.${OPENBALENA_HOST_NAME}
-    environment:
-      BALENA_HAPROXY_CRT: ${OPENBALENA_ROOT_CRT}
-      BALENA_HAPROXY_KEY: ${OPENBALENA_ROOT_KEY}
-      BALENA_ROOT_CA: ${OPENBALENA_ROOT_CA}
-      HAPROXY_HOSTNAME: ${OPENBALENA_HOST_NAME}

compose/template.yml

@@ -1,10 +0,0 @@
-# Project-specific config.
-#
-# All paths must be defined relative to `compose/services.yml` regardless of
-# the location of this file, i.e. refer to `my-open-balena-checkout/somedir`
-# as `../somedir`. This is because of the way docker-compose handles paths
-# when specifying multiple configs and open-balena always specifying
-# `compose/services.yml` as the "base" config.
-#
-# You may view the effective config with `scripts/compose config`.
-version: '2.1'

docker-compose.yml

@@ -0,0 +1,360 @@
+---
+version: '2.4'
+
+volumes:
+  builder-certs-ca: {}
+  builder-certs-client: {}
+  builder-data: {}
+  cert-manager-data: {}
+  certs-data: {}
+  db-data: {}
+  pki-data: {}
+  redis-data: {}
+  resin-data: {}
+  s3-data: {}
+
+x-default-healthcheck: &default-healthcheck
+  test: /usr/src/app/docker-hc
+  interval: 45s
+  timeout: 15s
+  retries: 3
+
+x-default-environment: &default-environment
+  # FIXME: hardcoded https://github.com/balena-io/open-balena-db/blob/master/create-resin-db.sh#L4
+  DB_NAME: resin
+  # FIXME: hardcoded https://github.com/balena-io/open-balena-db/blob/master/Dockerfile#L3-L4
+  DB_PASSWORD: docker
+  DB_USER: docker
+  LOG_LEVEL: DEBUG
+  PRODUCTION_MODE: 'false'
+
+x-default-healthcheck-trait: &with-default-healthcheck
+  healthcheck:
+    <<: *default-healthcheck
+
+x-default-volumes-trait: &with-default-volumes
+  volumes:
+    - certs-data:/certs
+    - resin-data:/balena
+
+x-default-privileges-trait: &with-default-privileges
+  cap_add:
+    - SYS_ADMIN
+    - SYS_RESOURCE
+  security_opt:
+    - apparmor=unconfined
+  tmpfs:
+    - /run
+    - /sys/fs/cgroup
+
+x-extended-privileges-trait: &with-extended-privileges
+  security_opt:
+    - apparmor=unconfined
+    - seccomp=unconfined
+
+x-all-privileges-trait: &with-all-privileges
+  privileged: true
+  cap_add:
+    - ALL
+
+x-network-privileges-trait: &with-network-privileges
+  cap_add:
+    - NET_ADMIN
+    - SYS_ADMIN
+    - SYS_RESOURCE
+
+x-base-service-definition: &base-service
+  restart: unless-stopped
+  # for docker-compose only, no effect on balenaCloud
+  env_file:
+    - .env
+  tty: 'true'  # send syastemd logs from containers to stdout
+
+services:
+  # https://github.com/balena-io/open-balena-api
+  api:
+    <<: [
+      *base-service,
+      *with-default-healthcheck,
+      *with-default-privileges,
+      *with-default-volumes,
+    ]
+    image: balena/open-balena-api:v22.2.3
+    depends_on:
+      - db
+      - redis
+      - s3
+    environment:
+      <<: *default-environment
+      CONTRACTS_PUBLIC_REPO_NAME: contracts
+      CONTRACTS_PUBLIC_REPO_OWNER: balena-io
+      DB_GENERAL_REPLICA_MAX_USES: 1000
+      DB_GENERAL_REPLICA_PORT: 5432
+      DB_HOST: db
+      DB_PORT: 5432
+      DB_STATE_REPLICA_MAX_USES: 1000
+      DB_STATE_REPLICA_PORT: 5432
+      DB_USER: docker
+      HOSTS_CONFIG: API_HOST:api,DB_HOST:db,DELTA_HOST:delta,HOST:api,REDIS_HOST:redis,TOKEN_AUTH_CERT_ISSUER:api,VPN_HOST:cloudlink,REGISTRY2_HOST:registry2
+      IMAGE_STORAGE_BUCKET: resin-production-img-cloudformation
+      IMAGE_STORAGE_ENDPOINT: s3.amazonaws.com
+      IMAGE_STORAGE_PREFIX: images
+      JSON_WEB_TOKEN_EXPIRY_MINUTES: 10080
+      NUM_WORKERS: 1
+      OAUTH_CALLBACK_PROTOCOL: https
+      PORT: 80
+      REDIS_HOST: redis:6379
+      REDIS_IS_CLUSTER: 'false'
+      TOKEN_AUTH_JWT_ALGO: ES256
+      TOKENS_CONFIG: API_SERVICE_API_KEY:hex,AUTH_RESINOS_REGISTRY_CODE:hex,COOKIE_SESSION_SECRET:hex,JSON_WEB_TOKEN_SECRET:hex,MIXPANEL_TOKEN:hex,SUPERUSER_PASSWORD:hex,TOKEN_AUTH_BUILDER_TOKEN:hex,VPN_GUEST_API_KEY:hex,VPN_SERVICE_API_KEY:hex,API_VPN_SERVICE_API_KEY:API_SERVICE_API_KEY,REGISTRY2_TOKEN:TOKEN_AUTH_BUILDER_TOKEN
+      TRUST_PROXY: 172.16.0.0/12
+      VPN_PORT: 443
+      WEBRESOURCES_S3_BUCKET: web-resources
+      WEBRESOURCES_S3_REGION: "us-east-1"  # this is required for minio
+
+  # https://github.com/balena-io/open-balena-registry
+  registry:
+    <<: [
+      *base-service,
+      *with-default-healthcheck,
+      *with-default-privileges,
+    ]
+    image: balena/open-balena-registry:v2.39.56
+    volumes:
+      - certs-data:/certs
+      - resin-data:/balena
+    depends_on:
+      - redis
+      - s3
+    environment:
+      COMMON_REGION: open-balena
+      HOSTS_CONFIG: REGISTRY2_HOST:registry2,REGISTRY2_TOKEN_AUTH_ISSUER:api,REGISTRY2_TOKEN_AUTH_REALM:api
+      REGISTRY2_CACHE_ADDR: redis:6379
+      REGISTRY2_CACHE_DB: 1
+      REGISTRY2_CACHE_ENABLED: 'true'
+      REGISTRY2_S3_BUCKET: registry-data
+      REGISTRY2_STORAGEPATH: /data
+      TOKENS_CONFIG: REGISTRY2_SECRETKEY:hex
+
+  # https://github.com/balena-io/open-balena-vpn
+  vpn:
+    <<: [
+      *base-service,
+      *with-default-healthcheck,
+      *with-default-volumes,
+      # privileges in order from minimum to maximum
+      *with-network-privileges,
+      *with-default-privileges,
+    ]
+    image: balena/open-balena-vpn:v11.30.9
+    depends_on:
+      - api
+    environment:
+      HOSTS_CONFIG: VPN_HOST:cloudlink
+      TOKENS_CONFIG: ','
+      VPN_HAPROXY_USEPROXYPROTOCOL: 'true'
+      VPN_PORT: 443
+      # ensure correct service instance IP is registered with the API
+      VPN_SERVICE_REGISTER_INTERFACE: eth0
+
+  # https://github.com/balena-io/open-balena-db
+  db:
+    <<: *base-service
+    image: balena/open-balena-db:v5.2.2
+    volumes:
+      - db-data:/var/lib/postgresql/data
+    environment:
+      <<: *default-environment
+    healthcheck:
+      test: pg_isready -U "$${DB_USER}" -d "$${DB_NAME}"
+
+  # https://github.com/balena-io/open-balena-s3
+  s3:
+    <<: [
+      *base-service,
+      *with-default-healthcheck,
+      *with-default-privileges,
+    ]
+    image: balena/open-balena-s3:v2.28.42
+    volumes:
+      - s3-data:/export
+      - certs-data:/certs
+      - resin-data:/balena
+    environment:
+      BUCKETS: registry-data;web-resources
+      HOSTS_CONFIG: REGISTRY2_S3_REGION_ENDPOINT:s3,WEBRESOURCES_S3_HOST:s3
+      TOKENS_CONFIG: REGISTRY2_S3_KEY:hex,REGISTRY2_S3_SECRET:hex,S3_MINIO_ACCESS_KEY:REGISTRY2_S3_KEY,S3_MINIO_SECRET_KEY:REGISTRY2_S3_SECRET,WEBRESOURCES_S3_ACCESS_KEY:REGISTRY2_S3_KEY,WEBRESOURCES_S3_SECRET_KEY:REGISTRY2_S3_SECRET
+
+  # https://hub.docker.com/_/redis
+  redis:
+    <<: *base-service
+    # https://redis.io/blog/what-redis-license-change-means-for-our-managed-service-providers/
+    image: redis:7.2-alpine
+    volumes:
+      - redis-data:/data
+    healthcheck:
+      <<: *default-healthcheck
+      test: echo INFO | redis-cli | grep redis_version
+
+  # https://github.com/balena-io/open-balena-haproxy
+  haproxy:
+    <<: [
+      *base-service,
+      *with-default-privileges,
+      *with-default-volumes,
+    ]
+    build: src/haproxy
+    sysctls:
+      # https://github.com/docker-library/haproxy/issues/160
+      net.ipv4.ip_unprivileged_port_start: 0
+    healthcheck:
+      <<: *default-healthcheck
+      test: true | openssl s_client -connect localhost:443
+    ports:
+      # haproxy/http
+      - "80:80/tcp"
+      # haproxy/tcp-router
+      - "443:443/tcp"
+      # haproxy/stats
+      - "1936:1936/tcp"
+    environment:
+      LOGLEVEL: info
+
+  # dynamically configure Docker network aliases based on DNS_TLD and ALIAS list
+  # allows DNS resolution from systemd-less images on the Docker network
+  haproxy-sidecar:
+    <<: *base-service
+    build: src/haproxy-sidecar
+    volumes:
+      - /var/run/docker.sock:/host/run/docker.sock
+    environment:
+      DOCKER_HOST: unix:///host/run/docker.sock
+      # resolved internally as {{service}}.{{dns-tld-without-balena-device-uuid}} to haproxy service
+      ALIASES: api,ca,cloudlink,db,delta,logs,redis,registry2,s3,stats,tunnel
+    labels:
+      io.balena.features.balena-socket: 1
+      io.balena.features.supervisor-api	: 1
+
+  # https://github.com/balena-io/cert-manager
+  # https://certbot.eff.org/docs/using.html
+  # https://certbot-dns-cloudflare.readthedocs.io/
+  cert-manager:
+    <<: *base-service
+    build: src/cert-manager
+    volumes:
+      - cert-manager-data:/etc/letsencrypt
+      - certs-data:/certs
+      - resin-data:/balena
+    depends_on:
+      - balena-ca
+    environment:
+      # wildcard certificate for reverse proxy
+      SSH_KEY_NAMES: ','
+      SUBJECT_ALTERNATE_NAMES: '*'
+    labels:
+      io.balena.features.balena-api: 1
+      io.balena.features.supervisor-api: 1
+
+  # https://github.com/balena-io/ca-private
+  # https://github.com/cloudflare/cfssl/blob/master/doc/api/intro.txt
+  balena-ca:
+    <<: *base-service
+    image: balena/ca-private:v0.0.14
+    volumes:
+      - pki-data:/pki
+      - certs-data:/certs
+      - resin-data:/balena
+    healthcheck:
+      test: curl --silent -I --fail localhost:8888
+      interval: 60s
+      timeout: 60s
+      retries: 10
+    labels:
+      # future expansion
+      io.balena.features.balena-api: 1
+      io.balena.features.supervisor-api: 1
+
+
+
+  # --- the following are not required for runtime operation of openBalena
+
+  # only relevant when running in AWS/EC2
+  tag-sidecar:
+    build: src/tag-sidecar
+    restart: no
+    environment:
+      ENABLED: 'true'
+    labels:
+      io.balena.features.balena-api: 1
+
+  # Software Under Test (SUT) tests orchestrator
+  sut:
+    <<: [
+      *base-service,
+      *with-extended-privileges,
+      *with-network-privileges,
+    ]
+    build: src/balena-tests
+    command: /usr/sbin/balena.sh
+    environment:
+      DOCKER_CERT_PATH: /docker-pki/client
+      DOCKER_HOST: docker:2376
+      DOCKER_TLS_VERIFY: 'true'
+      GUEST_IMAGE: /balena/balena.img
+    volumes:
+      - builder-certs-client:/docker-pki/client
+      - certs-data:/certs
+      - resin-data:/balena
+    labels:
+      io.balena.features.balena-api: 1
+      io.balena.features.supervisor-api: 1
+    restart: no
+
+  # virtual Device Under Test (DUT)
+  dut:
+    <<: [
+      *base-service,
+      *with-extended-privileges,
+      *with-network-privileges,
+    ]
+    # https://hub.docker.com/r/qemux/qemu-docker
+    # https://github.com/qemus/qemu-docker
+    build: src/test-device
+    entrypoint:
+      - /bin/sh
+      - -c
+    command:
+      - /usr/sbin/balena.sh
+    environment:
+      GUEST_IMAGE: /balena/balena.img
+      MEMORY: 3072M
+      CPU: 4
+    volumes:
+      - resin-data:/balena
+    devices:
+      - /dev/net/tun
+    restart: no
+
+  # https://hub.docker.com/_/docker
+  # pseudo(builder) service for balena-tests
+  docker:
+    <<: [
+      *base-service,
+      *with-extended-privileges,
+      *with-network-privileges,
+    ]
+    image: docker:dind
+    volumes:
+      - builder-data:/var/lib/docker
+      - builder-certs-ca:/docker-pki/ca
+      - builder-certs-client:/docker-pki/client
+      - /sys:/sys
+    environment:
+      DOCKER_TLS_CERTDIR: /docker-pki
+    healthcheck:
+      test: docker system info
+      interval: 60s
+      timeout: 60s
+      retries: 5
+    labels:
+      io.balena.features.sysfs: 1

docs/getting-started.md

@@ -0,0 +1,485 @@
+# openBalena Getting Started Guide
+
+This guide will walk you through the steps of deploying an openBalena server, that
+together with the balena CLI, will enable you to create and manage a fleet of devices
+running on your own infrastructure, on premises or in the cloud. The openBalena servers
+must be reachable by the devices, which is easiest to achieve with cloud providers like
+AWS, Google Cloud, Digital Ocean and others.
+
+This guide assumes a setup with two separate machines:
+
+- A _server_, running Linux with at least 2GB of memory. These instructions were tested
+  with Ubuntu 20.04, 22.04 and 24.04 x64 servers. The server must have a working
+  installation of [Docker Engine] and you must have root permissions.
+- A _local machine_, running Linux, Windows or macOS where the balena CLI runs (as a
+  client to the openBalena server). The local machine must also have a working
+  installation of [Docker] so that application images can be built and deployed to your
+  device. It is also possible to use [balenaEngine] on a [balenaOS] device instead of
+  Docker.
+
+Additionally, a _device type_ and compatible flash media supported by [balenaOS]
+(e.g. Raspberry Pi) are required to complete the provisioning demo. Ensure the correct
+power supply is available to power this device.
+
+## Domain Configuration
+
+The following DNS records must be configured to point to the openBalena server prior to
+configuration:
+
+```text
+api.mydomain.com
+ca.mydomain.com
+cloudlink.mydomain.com
+logs.mydomain.com
+ocsp.mydomain.com
+registry2.mydomain.com
+s3.mydomain.com
+tunnel.mydomain.com
+```
+
+Alternatively you may consider adding a single wildcard DNS record `*.mydomain.com`.
+
+Check with your Internet domain name registrar for instructions on how to obtain a domain
+name and configure records.
+
+## Install openBalena on the server
+
+1. First [Change cgroup version] to v1 for compatibility with systemd in containers on
+   modern Linux distributions, where cgroups v2 are enabled by default:
+
+    ```bash
+    source /etc/default/grub
+    sudo sed -i '/GRUB_CMDLINE_LINUX/d' /etc/default/grub
+    echo GRUB_CMDLINE_LINUX=$(printf '\"%s systemd.unified_cgroup_hierarchy=0\"\n' "${GRUB_CMDLINE_LINUX}") \
+      | sudo tee -a /etc/default/grub
+    sudo update-grub
+    sudo reboot
+    ```
+
+2. Ensure cgroups v2 is disabled
+
+    ```bash
+    if [ ! -f /sys/fs/cgroup/cgroup.controllers ]; then
+        echo "cgroups v2 is disabled"
+    else
+        echo "cgroups v2 is enabled"
+    fi
+    ```
+
+3. Now, install or update essential software:
+
+    ```bash
+    sudo apt-get update && sudo apt-get install -y make openssl git jq
+    ```
+
+4. Install Docker Engine
+
+    ```bash
+    which docker || curl -fsSL https://get.docker.com | sh -
+    ```
+
+5. Create a new user with appropriate permissions:
+
+    ```bash
+    sudo useradd -s /bin/bash -m -G docker,sudo balena
+    echo 'balena ALL=(ALL) NOPASSWD: ALL' | tee >/etc/sudoers.d/balena
+    ```
+
+6. Switch user:
+
+   ```bash
+   sudo su balena
+   ```
+
+7. Clone the openBalena repository and change directory:
+
+   ```bash
+   git clone https://github.com/balena-io/open-balena.git ~/open-balena
+   cd ~/open-balena
+   ```
+
+8. Start the server on your domain name:
+
+   ```bash
+   export DNS_TLD=mydomain.com
+   make up
+   ```
+
+   Note down `SUPERUSER_EMAIL` and `SUPERUSER_PASSWORD` values to be used later.
+
+9. Tail the logs of the containers with:
+
+   ```bash
+   docker compose logs -f api
+   ```
+
+   Replace `api` with the name of any one of the services from the [composition].
+
+10. The server can be stopped with:
+
+   ```bash
+   make down
+   ```
+
+   The server can also be restarted using `make restart`.
+
+To update openBalena, run:
+
+```bash
+make update
+```
+
+### Test the openBalena server
+
+To confirm that everything is running correctly, try a simple request from the local
+machine to the server after registering its CA certificate(s) with the host:
+
+```bash
+make self-signed
+make verify
+```
+
+Note, if you've previously stopped the server with `make down`, run `make up` again first.
+
+Congratulations! The openBalena server is up and running. The next step is to setup your
+local machine to use this server, provision a device and deploy a small project.
+
+### Install self-signed certificates on the local machine.
+
+The installation of the openBalena server produces a self-signed certificate by default,
+which must be trusted by all devices communicating with it. This type of configuration is
+not recommended for production deployments, skip to [SSL Configuration](#ssl-configuration)
+instead.
+
+The root CA bundle can be found at `.balena/ca-${DNS_TLD}.pem` on the server. Follow the
+steps below for your specific local machine platform after manually copying it across.
+
+#### Linux:
+
+```bash
+sudo cp ca.pem /usr/local/share/ca-certificates/
+sudo update-ca-certificates
+sudo systemctl restart docker
+```
+
+#### macOS:
+
+```bash
+sudo security add-trusted-cert -d \
+  -r trustRoot \
+  -k /Library/Keychains/System.keychain \
+  ca.pem
+
+curl http://localhost/engine/restart \
+  -H 'Content-Type: application/json' \
+  -d '{"openContainerView": true}' \
+  --unix-socket ~/Library/Containers/com.docker.docker/Data/backend.sock
+```
+
+#### Windows:
+
+```PowerShell
+certutil -addstore -f "ROOT" ca.pem
+Stop-Service -Name Docker
+Start-Service -Name Docker
+```
+
+### SSL Configuration
+
+opeBalena server now uses automatic SSL configuration via ACME [DNS-01] challenge[^1]. Support
+for the following DNS providers is currently implemented:
+
+* Cloudflare
+* Gandi
+
+#### Cloudflare
+
+Obtain a Cloudflare API token with write access to your openBalena domain name records:
+
+```bash
+export ACME_EMAIL=acme@mydomain.com
+export CLOUDFLARE_API_TOKEN={{token}}
+```
+
+#### Gandi
+
+Obtain a Gandi API token with write access to your openBalena domain name records:
+
+```bash
+export ACME_EMAIL=acme@mydomain.com
+export GANDI_API_TOKEN={{token}}
+```
+
+#### Re-configure and test the server
+
+```bash
+make auto-pki
+make verify
+```
+
+#### Custom SSL
+
+openBalena server also supports custom/manual TLS configuration. You must supply your own
+SSL certificate, private key and a full certificate signing chain. A wildcard SSL
+certificate covering the whole domain is recommended.
+
+1. After obtaining your certificate, run the following commands on openBalena server:
+
+```bash
+export HAPROXY_CRT="{{ base64 encoded server certificate }}"
+export ROOT_CA="{{ .. intermediate certificates }}"
+export HAPROXY_KEY="{{ .. private key }}"
+```
+
+Pipe the plaintext via `.. | openssl base64 -A` to encode.
+
+2. Re-configure and test the server:
+
+```bash
+make pki-custom
+make verify
+```
+
+### Install the balena CLI on the local machine
+
+Follow the [balena CLI installation instructions] to install the balena CLI on the local
+machine.
+
+By default, the CLI targets the balenaCloud servers at `balena-cloud.com`, and
+needs to be configured to target the openBalena server instead. Add the following
+line to the CLI's configuration file, replacing `"mydomain.com"` with the domain
+name of the openBalena server:
+
+```yaml
+balenaUrl: 'mydomain.com'
+```
+
+The CLI configuration file can be found at:
+
+- On Linux or macOS: `~/.balenarc.yml`
+- On Windows: `%UserProfile%\_balenarc.yml`
+
+If the file does not already exist, just create it. Alternatively, `BALENARC_BALENA_URL`
+environment variable can be set to point to `"mydomain.com"`.
+
+Wrapping up the CLI installation, set an environment variable that points to the
+root certificate copied previously on the local machine. This step is to ensure
+the CLI can securely interact with the openBalena server when running self-signed PKI.
+This step can be skipped if the server is operating with publicly trusted PKI.
+
+| Shell              | Command                                        |
+| ------------------ | ---------------------------------------------- |
+| bash               | `export NODE_EXTRA_CA_CERTS='/path/to/ca.pem'` |
+| Windows cmd.exe    | `set NODE_EXTRA_CA_CERTS=C:\path\to\ca.pem`    |
+| Windows PowerShell | `$Env:NODE_EXTRA_CA_CERTS="C:\path\to\ca.pem"` |
+
+### Deploy an application
+
+The commands below should be run on a terminal on the local machine (where the
+balena CLI is installed). Ensure that the `NODE_EXTRA_CA_CERTS` environment
+variable is set, as discussed above.
+
+#### Login to openBalena
+
+Run `balena login`, select `Credentials` and use `SUPERUSER_EMAIL` and
+`SUPERUSER_PASSWORD` generated during `make up` step to login to the openBalena server.
+At any time, `balena whoami` command may be used to check which server the CLI is
+authenticated with.
+
+#### Create an application
+
+Create a new application with `balena fleet create myApp`. Select the application's
+default device type with the interactive prompt. The examples in this guide assume
+a Raspberry Pi 3.
+
+An application contains devices that share the same architecture (such as ARM or Intel),
+and also contains code releases that are deployed to the devices. When a device is
+provisioned, it is added to an application, but can be migrated to another application at
+any time. There is no limit to the number of applications that can be created or to the
+number of devices that can be provisioned.
+
+At any time, the server can be queried for all the applications it knows about
+with the following command:
+
+```bash
+balena fleets
+ Id App name Slug        Device type  Device count Online devices
+ ── ──────── ─────────── ──────────── ──────────── ──────────────
+ 1  myApp    admin/myapp raspberrypi3 0            0
+```
+
+#### Provision a new device
+
+Once we have an application, it’s time to start provisioning devices. To do this,
+first download a [balenaOS] image for your device. For this example we are using a
+Raspberry Pi 3.
+
+Unzip the downloaded image and use the balena CLI to configure it:
+
+```bash
+balena os configure --dev --fleet myApp ~/Downloads/raspberrypi3-5.2.8-v16.1.10.img
+```
+
+Flash the configured image to an SD card using [Etcher] or balena CLI:
+
+```bash
+sudo balena local flash ~/Downloads/raspberrypi3-5.2.8-v16.1.10.img
+```
+
+Insert the SD card into the device and power it on. The device will register with the
+openBalena server and after about two minutes will be inspectable:
+
+```bash
+balena devices
+ID UUID    DEVICE NAME DEVICE TYPE  FLEET       STATUS IS ONLINE SUPERVISOR VERSION OS VERSION
+1  560dcc2 quiet-rock  raspberrypi3 admin/myapp Idle   true      16.1.10            balenaOS 5.2.8
+
+balena device 560dcc2
+== WANDERING RAIN
+ID:                    1
+DEVICE TYPE:           raspberrypi3
+STATUS:                idle
+IS ONLINE:             true
+IP ADDRESS:            192.168.1.42
+MAC ADDRESS:           B8:27:DE:AD:BE:EF
+FLEET:                 admin/myapp
+LAST SEEN:             1977-08-20T14:29:00.042Z
+UUID:                  560dcc24b221c8a264d5bd981284801f
+COMMIT:                N/a
+SUPERVISOR VERSION:    16.1.10
+IS WEB ACCESSIBLE:     false
+OS VERSION:            balenaOS 5.2.8
+DASHBOARD URL:         https://dashboard.mydomain.com/devices/560dcc24b221c8a264d5bd981284801f/summary
+CPU USAGE PERCENT:     2
+CPU TEMP C:            39
+CPU ID:                00000000335956af
+MEMORY USAGE MB:       140
+MEMORY TOTAL MB:       971
+MEMORY USAGE PERCENT:  14
+STORAGE BLOCK DEVICE:  /dev/mmcblk0p6
+STORAGE USAGE MB:      76
+STORAGE TOTAL MB:      14121
+STORAGE USAGE PERCENT: 1
+```
+
+Note, even though the dashboard URL is populated, there is no dashboard service in
+openBalena.
+
+It's time to deploy code to the device.
+
+#### Deploy a project
+
+Application release images are built on the local machine using the balena CLI. Ensure the
+root certificate has been correctly installed on the local machine, as discussed above.
+
+Let's create a trivial project that logs "Idling...". On an empty directory, create a new
+file named `Dockerfile.template` with the following contents:
+
+```dockerfile
+FROM balenalib/%%BALENA_MACHINE_NAME%%-alpine
+
+CMD [ "balena-idle" ]
+```
+
+Then build and deploy the project with:
+
+```bash
+balena deploy --noparent-check myApp
+```
+
+The project will have been successfully built when a friendly unicorn appears in the
+terminal:
+
+```bash
+[Info]    No "docker-compose.yml" file found at "~/open-balena/balena-idle"
+[Info]    Creating default composition with source: "~/open-balena/balena-idle"
+[Info]    Everything is up to date (use --build to force a rebuild)
+[Info]    Creating release...
+[Info]    Pushing images to registry...
+[Info]    Saving release...
+[Success] Deploy succeeded!
+[Success] Release: 50be7bdb0ea6819c91a5dd7bcd7635ad
+
+                            \
+                             \
+                              \\
+                               \\
+                                >\/7
+                            _.-(6'  \
+                           (=___._/` \
+                                )  \ |
+                               /   / |
+                              /    > /
+                             j    < _\
+                         _.-' :      ``.
+                         \ r=._\        `.
+                        <`\\_  \         .`-.
+                         \ r-7  `-. ._  ' .  `\
+                          \`,      `-.`7  7)   )
+                           \/         \|  \'  / `-._
+                                      ||    .'
+                                       \\  (
+                                        >\  >
+                                    ,.-' >.'
+                                   <.'_.''
+                                     <'
+```
+
+This command packages up the local directory, creates a new Docker image from it and
+pushes it to the openBalena server. In turn, the server will deploy it to all provisioned
+devices and within a couple of minutes, they will all run the new release. Logs can be
+viewed with:
+
+```bash
+balena logs --tail 560dcc2
+[Logs]    [2024-05-02T15:59:31.383Z] Supervisor starting
+[Logs]    [2024-05-02T15:59:37.552Z] Applying configuration change {"SUPERVISOR_VPN_CONTROL":"true"}
+[Logs]    [2024-05-02T15:59:37.599Z] Applied configuration change {"SUPERVISOR_VPN_CONTROL":"true"}
+[Logs]    [2024-05-02T15:59:40.331Z] Creating network 'default'
+[Logs]    [2024-05-02T16:11:15.331Z] Supervisor starting
+[Logs]    [2024-05-02T16:44:08.199Z] Creating volume 'resin-data'
+[Logs]    [2024-05-02T16:44:08.572Z] Downloading image 'registry2.mydomain.com/v2/…
+…
+[Logs]    [2024-05-02T16:44:37.200Z] [main] Idling...
+[Logs]    [2024-05-02T16:44:37.200Z] [main] Idling...
+```
+
+Enjoy Balenafying All the Things!
+
+## Next steps
+
+- Try out [local mode], which allows you to build and sync code to your device locally for
+  rapid development.
+- Develop an application with [multiple containers] to provide a more modular approach to
+  application management.
+- Manage your device fleet with the use of [configuration] and [environment] variables.
+- Explore our [example projects] to give you an idea of more things you can do with
+  balena.
+- If you find yourself stuck or confused, help is just [a click away].
+- Pin selected devices to selected code releases using [sample scripts].
+- To change the superuser password after setting the credentials, follow this [forum post]
+
+
+[^1]: If DNS validation is not an option, [acme.sh] or [certbot] can be used to manually
+issue a certificate, which can then be set using the [custom SSL](#custom-ssl) workflow.
+
+
+[local mode]: https://www.balena.io/docs/learn/develop/local-mode
+[multiple containers]: https://www.balena.io/docs/learn/develop/multicontainer
+[configuration]: https://www.balena.io/docs/learn/manage/configuration
+[environment]: https://www.balena.io/docs/learn/manage/serv-vars
+[example projects]: https://balena.io/blog/tags/etcher-featured
+[a click away]: https://www.balena.io/support
+[sample scripts]: https://github.com/balena-io-examples/staged-releases
+[forum post]: https://forums.balena.io/t/upate-superuser-password/4738/6
+[balena CLI installation instructions]: https://github.com/balena-io/balena-cli/blob/master/INSTALL.md
+[Etcher]: https://balena.io/etcher
+[balenaOS]: https://balena.io/os/#download
+[balenaEngine]: https://www.balena.io/engine
+[Docker]: https://docs.docker.com/get-docker
+[Docker Engine]: https://docs.docker.com/engine/install
+[Change cgroup version]: https://docs.docker.com/config/containers/runmetrics/#changing-cgroup-version
+[composition]: https://github.com/balena-io/open-balena/blob/master/docker-compose.yml
+[DNS-01]: https://letsencrypt.org/docs/challenge-types/#dns-01-challenge
+[acme.sh]: https://github.com/acmesh-official/acme.sh
+[certbot]: https://certbot.eff.org/

haproxy/Dockerfile

@@ -1,6 +0,0 @@
-FROM haproxy:1.8-alpine
-
-COPY haproxy.cfg /usr/local/etc/haproxy/haproxy.cfg
-COPY entry.sh /open-balena-entry
-
-CMD /open-balena-entry

haproxy/entry.sh

@@ -1,11 +0,0 @@
-#!/bin/sh
-set -e
-
-HAPROXY_CHAIN=/etc/ssl/private/open-balena.pem
-mkdir -p "$(dirname "${HAPROXY_CHAIN}")"
-(
-    echo "${BALENA_HAPROXY_CRT}" | base64 -d
-    echo "${BALENA_HAPROXY_KEY}" | base64 -d
-    echo "${BALENA_ROOT_CA}" | base64 -d
-) > "${HAPROXY_CHAIN}"
-exec haproxy -f /usr/local/etc/haproxy/haproxy.cfg

haproxy/haproxy.cfg

@@ -1,110 +0,0 @@
-global
-  tune.ssl.default-dh-param 1024
-
-defaults
-  timeout connect 5000
-  timeout client 50000
-  timeout server 50000
-
-frontend http-in
-  mode http
-  option forwardfor
-  bind *:80
-  reqadd X-Forwarded-Proto:\ http
-
-  acl host_api hdr_dom(host) -i "api.${HAPROXY_HOSTNAME}"
-  use_backend backend_api if host_api
-
-  acl host_registry hdr_dom(host) -i "registry.${HAPROXY_HOSTNAME}"
-  use_backend backend_registry if host_registry
-
-  acl host_vpn hdr_dom(host) -i "vpn.${HAPROXY_HOSTNAME}"
-  use_backend backend_vpn if host_vpn
-
-  acl host_s3 hdr_dom(host) -i "s3.${HAPROXY_HOSTNAME}"
-  use_backend backend_s3 if host_s3
-
-frontend ssl-in
-  mode tcp
-  bind *:443
-  tcp-request inspect-delay 2s
-  tcp-request content accept if { req.ssl_hello_type 1 }
-
-  acl is_ssl req.ssl_ver 2:3.4
-  use_backend redirect-to-https-in if is_ssl
-  use_backend vpn-devices if !is_ssl
-
-backend redirect-to-https-in
-  mode tcp
-  balance roundrobin
-  server localhost 127.0.0.1:444 send-proxy-v2
-
-frontend https-in
-  mode http
-  option forwardfor
-  bind 127.0.0.1:444 ssl crt /etc/ssl/private/open-balena.pem accept-proxy
-  reqadd X-Forwarded-Proto:\ https
-
-  acl host_api hdr_dom(host) -i "api.${HAPROXY_HOSTNAME}"
-  use_backend backend_api if host_api
-
-  acl host_registry hdr_dom(host) -i "registry.${HAPROXY_HOSTNAME}"
-  use_backend backend_registry if host_registry
-
-  acl host_vpn hdr_dom(host) -i "vpn.${HAPROXY_HOSTNAME}"
-  use_backend backend_vpn if host_vpn
-
-  acl host_s3 hdr_dom(host) -i "s3.${HAPROXY_HOSTNAME}"
-  use_backend backend_s3 if host_s3
-
-backend backend_api
-  mode http
-  option forwardfor
-  balance roundrobin
-  server resin_api_1 api:80 check port 80
-
-backend backend_registry
-  mode http
-  option forwardfor
-  balance roundrobin
-  server resin_registry_1 registry:80 check port 80
-
-backend backend_vpn
-  mode http
-  option forwardfor
-  balance roundrobin
-  server resin_vpn_1 vpn:80 check port 80
-
-backend backend_s3
-  mode http
-  option forwardfor
-  balance roundrobin
-
-backend vpn-devices
-  mode tcp
-  server resin_vpn_1 vpn:443 send-proxy-v2 check-send-proxy port 443
-
-frontend db
-  mode tcp
-  bind *:5432
-  default_backend backend_db
-  timeout client 1h
-
-backend backend_db
-  mode tcp
-  server resin_db_1 db:5432 check port 5432
-
-frontend redis
-  mode tcp
-  bind *:6379
-  default_backend backend_redis
-  timeout client 1h
-
-backend backend_redis
-  mode tcp
-  server resin_redis_1 redis:6379 check port 6379
-
-listen vpn-tunnel
-  mode tcp
-  bind *:3128
-  server balena_vpn vpn:3128 check port 3128

repo.yml

@@ -1,2 +1,13 @@
-type: 'generic'
-reviewers: 1
+---
+type: generic
+upstream:
+  - repo: open-balena-api
+    url: https://github.com/balena-io/open-balena-api
+  - repo: open-balena-vpn
+    url: https://github.com/balena-io/open-balena-vpn
+  - repo: open-balena-registry
+    url: https://github.com/balena-io/open-balena-registry
+  - repo: open-balena-db
+    url: https://github.com/balena-io/open-balena-db
+  - repo: open-balena-s3
+    url: https://github.com/balena-io/open-balena-s3

scripts/_keyid.js

@@ -1,79 +0,0 @@
-'use strict';
-
-var crypto = require('crypto');
-var fs = require('fs');
-
-var base32 = (function() {
-  // Extracted from https://github.com/chrisumbel/thirty-two
-  // to avoid having to install packages for this script.
-  var charTable = "ABCDEFGHIJKLMNOPQRSTUVWXYZ234567";
-  var byteTable = [
-    0xff, 0xff, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f,
-    0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
-    0xff, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06,
-    0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e,
-    0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16,
-    0x17, 0x18, 0x19, 0xff, 0xff, 0xff, 0xff, 0xff,
-    0xff, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06,
-    0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e,
-    0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16,
-    0x17, 0x18, 0x19, 0xff, 0xff, 0xff, 0xff, 0xff
-  ];
-
-  function quintetCount(buff) {
-    var quintets = Math.floor(buff.length / 5);
-    return buff.length % 5 == 0 ? quintets: quintets + 1;
-  }
-
-  return function(plain) {
-    if (!Buffer.isBuffer(plain)) {
-      plain = new Buffer(plain);
-    }
-    var i = 0;
-    var j = 0;
-    var shiftIndex = 0;
-    var digit = 0;
-    var encoded = new Buffer(quintetCount(plain) * 8);
-
-    /* byte by byte isn't as pretty as quintet by quintet but tests a bit
-      faster. will have to revisit. */
-    while(i < plain.length) {
-      var current = plain[i];
-
-      if(shiftIndex > 3) {
-        digit = current & (0xff >> shiftIndex);
-        shiftIndex = (shiftIndex + 5) % 8;
-        digit = (digit << shiftIndex) | ((i + 1 < plain.length) ?
-          plain[i + 1] : 0) >> (8 - shiftIndex);
-        i++;
-      } else {
-        digit = (current >> (8 - (shiftIndex + 5))) & 0x1f;
-        shiftIndex = (shiftIndex + 5) % 8;
-        if(shiftIndex == 0) i++;
-      }
-
-      encoded[j] = charTable.charCodeAt(digit);
-      j++;
-    }
-
-    for (i = j; i < encoded.length; i++) {
-      encoded[i] = 0x3d; //'='.charCodeAt(0)
-    }
-    return encoded;
-  }
-})();
-
-function joseKeyId(der) {
-  var hasher = crypto.createHash('sha256');
-  hasher.update(der);
-  var b32 = base32(hasher.digest().slice(0, 30)).toString('ascii');
-  var chunks = [];
-  for (var i = 0; i < b32.length; i += 4) {
-    chunks.push(b32.substr(i, 4));
-  }
-  return chunks.join(':');
-}
-
-var derFilePath = process.argv[2];
-var der = fs.readFileSync(derFilePath);
-process.stdout.write(joseKeyId(der));

scripts/_realpath

@@ -1,24 +0,0 @@
-#!/bin/bash -e
-
-REALPATH=
-REALPATHS=(
-    'realpath'
-    'grealpath'
-    'greadlink -f'
-)
-for cmd in "${REALPATHS[@]}"; do
-if command -v "${cmd%% *}" &>/dev/null; then
-    REALPATH="${cmd}"
-    break
-fi
-done
-
-if [ -z "${REALPATH}" ]; then
-    local RED=`tput setaf 1`
-    echo "${RED}ERROR: Unable to find suitable command for realpath."
-    exit 1
-fi
-
-realpath() {
-  echo $(command ${REALPATH} "$@")
-}

scripts/compose

@@ -1,26 +0,0 @@
-#!/bin/bash -e
-
-source "${BASH_SOURCE%/*}/_realpath"
-
-CMD="$(realpath "$0")"
-DIR="$(dirname "${CMD}")"
-BASE_DIR="$(dirname "${DIR}")"
-CONFIG_DIR="${BASE_DIR}/config"
-
-echo_bold() {
-  printf "\\033[1m%s\\033[0m\\n" "$@"
-}
-
-ENV_FILE="${CONFIG_DIR}/activate"
-if [ ! -f "$ENV_FILE" ]; then
-  echo_bold 'No configuration found; please create one first with: ./scripts/quickstart'
-  echo_bold 'See README.md for help.'
-  exit 1
-fi
-
-# shellcheck source=/dev/null
-source "${ENV_FILE}"; docker-compose \
-  --project-name 'openbalena' \
-  -f "${BASE_DIR}/compose/services.yml" \
-  -f "${CONFIG_DIR}/docker-compose.yml" \
-  "$@"

scripts/create-superuser

@@ -1,36 +0,0 @@
-#!/bin/bash -e
-
-usage() {
-  echo "usage: $0 EMAIL PASSWORD"
-  echo
-  echo 'Create the superuser account with the given email and password.'
-  echo
-  echo 'The instance must already be running in the background. You can '
-  echo 'start it with: ./scripts/compose up -d'
-}
-
-if [[ -z "$1" || -z "$2" ]]; then
-  usage
-  exit 1
-fi
-
-echo_bold() {
-  printf "\\033[1m%s\\033[0m\\n" "${@}"
-}
-
-source "${BASH_SOURCE%/*}/_realpath"
-
-CMD="$(realpath "$0")"
-DIR="$(dirname "${CMD}")"
-FIG="${DIR}/compose"
-
-EMAIL="$1"
-PASSWORD="$2"
-
-"${FIG}" exec api /bin/bash -c \
-  'export $(grep -v "^#" config/env | xargs -d "\n"); node index.js create-superuser root '${EMAIL}' '${PASSWORD}'' \
-  >/dev/null \
-  || (echo 'Failed to create superuser; please ensure the instance is running and that no superuser has been created before.' && exit 1)
-
-echo_bold "==> Success! Superuser created with email: ${EMAIL}"
-echo "  - You may now login with: balena login --credentials --email ${EMAIL}"

scripts/gen-root-ca

@@ -1,35 +0,0 @@
-#!/bin/bash -e
-
-usage() {
-  echo "usage: $0 COMMON_NAME [OUT]"
-  echo
-  echo "  COMMON_NAME   the domain name the certificate is valid for, eg. example.com"
-  echo "  OUT           path to output directory generated files will be placed in"
-  echo
-}
-
-if [ -z "$1" ]; then
-  usage
-  exit 1
-fi
-
-CMD="$(realpath "$0")"
-DIR="$(dirname "${CMD}")"
-
-CN="$1"
-OUT="$(realpath "${2:-.}")"
-
-# shellcheck source=scripts/ssl-common.sh
-source "${DIR}/ssl-common.sh"
-
-ROOT_CA="${ROOT_PKI}/ca.crt"
-
-if [ ! -f $ROOT_CA ]; then
-  # Create a secret key and CA file for the self-signed CA
-  "$easyrsa_bin" --pki-dir="${ROOT_PKI}" init-pki 2>/dev/null
-  "$easyrsa_bin" --pki-dir="${ROOT_PKI}" --days="${CA_EXPIRY_DAYS}" --req-cn="ca.${CN}" build-ca nopass 2>/dev/null
-
-  # update indexes and generate CRLs
-  "$easyrsa_bin" --pki-dir="${ROOT_PKI}" update-db 2>/dev/null
-  "$easyrsa_bin" --pki-dir="${ROOT_PKI}" gen-crl 2>/dev/null
-fi

scripts/gen-root-cert

@@ -1,36 +0,0 @@
-#!/bin/bash -e
-
-usage() {
-  echo "usage: $0 COMMON_NAME [OUT]"
-  echo
-  echo "  COMMON_NAME   the domain name the certificate is valid for, eg. example.com"
-  echo "  OUT           path to output directory generated files will be placed in"
-  echo
-}
-
-if [ -z "$1" ]; then
-  usage
-  exit 1
-fi
-
-CMD="$(realpath "$0")"
-DIR="$(dirname "${CMD}")"
-
-CN="$1"
-OUT="$(realpath "${2:-.}")"
-
-# shellcheck source=scripts/ssl-common.sh
-source "${DIR}/ssl-common.sh"
-
-ROOT_CRT="${ROOT_PKI}"'/issued/*.'"${CN}"'.crt'
-ROOT_KEY="${ROOT_PKI}"'/private/*.'"${CN}"'.key'
-
-if [ ! -f $ROOT_CRT ] || [ ! -f $ROOT_KEY ]; then
-  rm -f $ROOT_CRT $ROOT_KEY
-  # generate default CSR and sign (root + wildcard)
-  "$easyrsa_bin" --pki-dir="${ROOT_PKI}" --days="${CRT_EXPIRY_DAYS}" --subject-alt-name="DNS:*.${CN}" build-server-full "*.${CN}" nopass 2>/dev/null
-
-  # update indexes and generate CRLs
-  "$easyrsa_bin" --pki-dir="${ROOT_PKI}" update-db 2>/dev/null
-  "$easyrsa_bin" --pki-dir="${ROOT_PKI}" gen-crl 2>/dev/null
-fi;

scripts/gen-token-auth-cert

@@ -1,53 +0,0 @@
-#!/bin/bash -e
-
-usage() {
-  echo "usage: $0 COMMON_NAME [OUT]"
-  echo
-  echo "  COMMON_NAME   the domain name the certificate is valid for, eg. example.com"
-  echo "  OUT           path to output directory generated files will be placed in"
-  echo
-}
-
-if [ -z "$1" ]; then
-  usage
-  exit 1
-fi
-
-CMD="$(realpath "$0")"
-DIR="$(dirname "${CMD}")"
-
-CN="$1"
-OUT="$(realpath "${2:-.}")"
-
-# shellcheck source=scripts/ssl-common.sh
-source "${DIR}/ssl-common.sh"
-
-CERT_DIR="${OUT}/api"
-CERT_FILE="${CERT_DIR}/api.${CN}"
-
-keyid() {
-  # NodeJS is installed as `nodejs` in some distros, `node` in others.
-  node_bin="$(command -v nodejs 2>/dev/null || command -v node 2>/dev/null || true)"
-  if [ -z "$node_bin" ]; then
-    echo >&2 'NodeJS is required but not installed. Aborting.'
-    exit 1
-  fi
-  # Recent Node versions complain about `new Buffer()` being deprecated
-  # but the alternative is not available to older versions. Silence the
-  # warning but use the deprecated form to allow greater compatibility.
-  "$node_bin" --no-deprecation "${DIR}/_keyid.js" "$1"
-}
-
-JWT_CRT="${CERT_FILE}.crt"
-JWT_KEY="${CERT_FILE}.pem"
-JWT_KID="${CERT_FILE}.kid"
-
-if [ ! -f $JWT_CRT ] || [ ! -f $JWT_KEY ] || [ ! -f $JWT_KID ]; then
-  rm -f $JWT_CRT $JWT_KEY $JWT_KID
-  mkdir -p "${CERT_DIR}"
-  openssl ecparam -name prime256v1 -genkey -noout -out "${JWT_KEY}" 2>/dev/null
-  openssl req -x509 -new -nodes -days "${CRT_EXPIRY_DAYS}" -key "${JWT_KEY}" -subj "/CN=api.${CN}" -out "${JWT_CRT}" 2>/dev/null
-  openssl ec -in "${JWT_KEY}" -pubout -outform DER -out "${CERT_FILE}.der" 2>/dev/null
-  keyid "${CERT_FILE}.der" >"${JWT_KID}"
-  rm "${CERT_FILE}.der"
-fi

scripts/gen-vpn-certs

@@ -1,55 +0,0 @@
-#!/bin/bash -e
-
-usage() {
-  echo "usage: $0 COMMON_NAME [OUT]"
-  echo
-  echo "  COMMON_NAME   the domain name the certificate is valid for, eg. example.com"
-  echo "  OUT           path to output directory generated files will be placed in"
-  echo
-}
-
-if [ -z "$1" ]; then
-  usage
-  exit 1
-fi
-
-CMD="$(realpath "$0")"
-DIR="$(dirname "${CMD}")"
-
-CN="$1"
-OUT="$(realpath "${2:-.}")"
-
-# shellcheck source=scripts/ssl-common.sh
-source "${DIR}/ssl-common.sh"
-
-VPN_PKI="$(realpath "${OUT}/vpn")"
-VPN_CA="${VPN_PKI}/ca.crt"
-VPN_CRT="${VPN_PKI}/issued/vpn.${CN}.crt"
-VPN_KEY="${VPN_PKI}/private/vpn.${CN}.key"
-VPN_DH="${VPN_PKI}/dh.pem"
-
-if [ ! -f $VPN_CA ] || [ ! -f $VPN_CRT ] || [ ! -f $VPN_KEY ] || [ ! -f $VPN_DH ]; then
-
-  rm -f $VPN_CA $VPN_CRT $VPN_DH $VPN_KEY
-
-  # generate VPN sub-CA
-  "$easyrsa_bin" --pki-dir="${VPN_PKI}" init-pki &>/dev/null
-  "$easyrsa_bin" --pki-dir="${VPN_PKI}" --days="${CA_EXPIRY_DAYS}" --req-cn="vpn-ca.${CN}" build-ca nopass subca 2>/dev/null
-
-  # import sub-CA CSR into root PKI, sign, and copy back to vpn PKI
-  "$easyrsa_bin" --pki-dir="${ROOT_PKI}" import-req "${VPN_PKI}/reqs/ca.req" "vpn-ca" 2>/dev/null
-  "$easyrsa_bin" --pki-dir="${ROOT_PKI}" sign-req ca "vpn-ca" 2>/dev/null
-  cp "${ROOT_PKI}/issued/vpn-ca.crt" "${VPN_PKI}/ca.crt"
-
-  # generate and sign vpn server certificate
-  "$easyrsa_bin" --pki-dir="${VPN_PKI}" --days="${CRT_EXPIRY_DAYS}" build-server-full "vpn.${CN}" nopass 2>/dev/null
-
-  # generate vpn dhparams (keysize of 2048 will do, 4096 can wind up taking hours to generate)
-  "$easyrsa_bin" --pki-dir="${VPN_PKI}" --keysize=2048 gen-dh 2>/dev/null
-
-  # update indexes and generate CRLs
-  "$easyrsa_bin" --pki-dir="${ROOT_PKI}" update-db 2>/dev/null
-  "$easyrsa_bin" --pki-dir="${VPN_PKI}" update-db 2>/dev/null
-  "$easyrsa_bin" --pki-dir="${ROOT_PKI}" gen-crl 2>/dev/null
-  "$easyrsa_bin" --pki-dir="${VPN_PKI}" gen-crl 2>/dev/null
-fi

scripts/make-env

@@ -1,97 +0,0 @@
-#!/bin/bash -e
-
-usage() {
-  echo "usage: $0"
-  echo
-  echo "Required Variables:"
-  echo
-  echo "  DOMAIN"
-  echo "  ROOT_CA    Path to root CA certificate"
-  echo "  ROOT_CRT   Path to root/wildcard certificate"
-  echo "  ROOT_KEY   Path to root/wildcard private key"
-  echo "  JWT_CRT    Path to Token Auth certificate"
-  echo "  JWT_KEY    Path to Token Auth private key"
-  echo "  JWT_KID    Path to KeyID for the Token Auth certificate"
-  echo "  VPN_CA     Path to the VPN sub-CA certificate"
-  echo "  VPN_CRT    Path to the VPN server certificate"
-  echo "  VPN_KEY    Path to the VPN server private key"
-  echo "  VPN_DH     Path to the VPN server Diffie Hellman parameters"
-  echo "  SUPERUSER_EMAIL     Email address of the superuser"
-  echo "  SUPERUSER_PASSWORD  Password of the superuser"
-  echo
-}
-
-for var in DOMAIN ROOT_CA ROOT_CRT ROOT_KEY JWT_CRT JWT_KEY JWT_KID VPN_CA VPN_CRT VPN_KEY VPN_DH SUPERUSER_EMAIL SUPERUSER_PASSWORD; do
-  if [ -z "${!var-}" ]; then
-    usage
-    exit 1
-  fi
-done
-
-randstr() {
-  LC_CTYPE=C tr -dc A-Za-z0-9 < /dev/urandom | fold -w "${1:-32}" | head -n 1
-}
-
-b64encode() {
-    echo "$@" | base64 --wrap=0 2>/dev/null || echo "$@" | base64 --break=0 2>/dev/null
-}
-
-b64file() {
-  b64encode "$(cat "$@")"
-}
-
-VPN_CONFIG=$(cat <<STR
-client
-remote vpn.$DOMAIN 443
-resolv-retry infinite
-
-remote-cert-tls server
-ca /etc/openvpn/ca.crt
-auth-user-pass /var/volatile/vpn-auth
-auth-retry none
-script-security 2
-up /etc/openvpn-misc/upscript.sh
-up-restart
-down /etc/openvpn-misc/downscript.sh
-
-comp-lzo
-dev resin-vpn
-dev-type tun
-proto tcp
-nobind
-
-persist-key
-persist-tun
-verb 3
-user openvpn
-group openvpn
-
-STR
-)
-
-cat <<STR
-export OPENBALENA_PRODUCTION_MODE=false
-export OPENBALENA_COOKIE_SESSION_SECRET=$(randstr 32)
-export OPENBALENA_HOST_NAME=$DOMAIN
-export OPENBALENA_JWT_SECRET=$(randstr 32)
-export OPENBALENA_RESINOS_REGISTRY_CODE=$(randstr 32)
-export OPENBALENA_ROOT_CA=$(b64file "${ROOT_CA}")
-export OPENBALENA_ROOT_CRT=$(b64file "${ROOT_CRT}")
-export OPENBALENA_ROOT_KEY=$(b64file "${ROOT_KEY}")
-export OPENBALENA_TOKEN_AUTH_BUILDER_TOKEN=$(randstr 64)
-export OPENBALENA_TOKEN_AUTH_PUB=$(b64file "$JWT_CRT")
-export OPENBALENA_TOKEN_AUTH_KEY=$(b64file "$JWT_KEY")
-export OPENBALENA_TOKEN_AUTH_KID=$(b64file "$JWT_KID")
-export OPENBALENA_VPN_CA=$(b64file "$VPN_CA")
-export OPENBALENA_VPN_CA_CHAIN=$(b64file "$ROOT_CA" "$VPN_CA")
-export OPENBALENA_VPN_CONFIG=$(b64encode "$VPN_CONFIG")
-export OPENBALENA_VPN_SERVER_CRT=$(b64file "$VPN_CRT")
-export OPENBALENA_VPN_SERVER_KEY=$(b64file "$VPN_KEY")
-export OPENBALENA_VPN_SERVER_DH=$(b64file "$VPN_DH")
-export OPENBALENA_VPN_SERVICE_API_KEY=$(randstr 32)
-export OPENBALENA_API_VPN_SERVICE_API_KEY=$(randstr 32)
-export OPENBALENA_REGISTRY_SECRET_KEY=$(randstr 32)
-export OPENBALENA_SSH_AUTHORIZED_KEYS=
-export OPENBALENA_SUPERUSER_EMAIL=$SUPERUSER_EMAIL
-export OPENBALENA_SUPERUSER_PASSWORD=$(printf "%q" "${SUPERUSER_PASSWORD}")
-STR

scripts/patch-hosts

@@ -1,30 +0,0 @@
-#!/bin/bash -e
-
-usage() {
-  echo "usage: $0 DOMAIN"
-  echo
-  echo "  DOMAIN  the domain name to add host entries for, eg. example.com"
-  echo
-}
-
-if [ -z "$1" ]; then
-  usage
-  exit 1
-fi
-
-SERVICES="api registry vpn db s3 redis"
-DOMAIN="$1"
-
-# We need sudo to write to /etc/hosts, so first write to a temp file and then
-# append all entries to hosts file.
-tmp="$(mktemp --tmpdir openbalena.XXXX)"
-for service in $SERVICES; do
-  name="${service}.${DOMAIN}"
-  if ! grep "\\s$name" /etc/hosts >/dev/null 2>&1 ; then
-    echo "adding $name"
-    echo "127.0.0.1 $name" >>"${tmp}"
-  fi
-done
-# shellcheck disable=SC2024
-sudo tee -a /etc/hosts >/dev/null <"${tmp}"
-rm -f "${tmp}"

scripts/quickstart

@@ -1,114 +0,0 @@
-#!/bin/bash -e
-
-BLACK=`tput setaf 0`
-RED=`tput setaf 1`
-GREEN=`tput setaf 2`
-YELLOW=`tput setaf 3`
-BLUE=`tput setaf 4`
-MAGENTA=`tput setaf 5`
-CYAN=`tput setaf 6`
-WHITE=`tput setaf 7`
-
-BOLD=`tput bold`
-RESET=`tput sgr0`
-
-# for macos machines, we need proper OpenSSL...
-OPENSSL_VERSION=$(openssl version -v)
-if [[ "${OPENSSL_VERSION}" =~ ^LibreSSL.*$ ]]; then
-  echo -e "${RED}ERROR: You may not have a compatible OpenSSL version (${OPENSSL_VERSION}). Please install OpenSSL version 1.0.2q or above.${RESET}"
-  exit 1
-fi
-
-source "${BASH_SOURCE%/*}/_realpath"
-
-CMD="$(realpath "$0")"
-DIR="$(dirname "${CMD}")"
-BASE_DIR="$(dirname "${DIR}")"
-CONFIG_DIR="${BASE_DIR}/config"
-CERTS_DIR="${CONFIG_DIR}/certs"
-
-DOMAIN=openbalena.local
-
-usage() {
-  echo "usage: $0 [-h] [-p] [-d DOMAIN] -U EMAIL -P PASSWORD"
-  echo
-  echo "  -p           patch hosts - patch the host /etc/hosts file"
-  echo "  -d DOMAIN    the domain name this deployment will run as, eg. example.com. Default is 'openbalena.local'"
-  echo "  -U EMAIL     the email address of the superuser account, used to login to your install from the Balena CLI"
-  echo "  -P PASSWORD  the password to use for the superuser account."
-  echo
-}
-
-show_help=false
-patch_hosts=false
-while getopts ":hpxd:U:P:" opt; do
-  case "${opt}" in
-    h) show_help=true;;
-    p) patch_hosts=true;;
-    x) set -x;;
-    d) DOMAIN="${OPTARG}";;
-    U) SUPERUSER_EMAIL="${OPTARG}";;
-    P) SUPERUSER_PASSWORD="${OPTARG}";;
-    *)
-      echo "Invalid argument: -${OPTARG}"
-      usage
-      exit 1
-      ;;
-  esac
-done
-shift $((OPTIND-1))
-
-if [ -z "${SUPERUSER_EMAIL}" ] || [ -z "${SUPERUSER_PASSWORD}" ]; then
-  usage
-  exit 1
-fi
-
-if [ "$show_help" = "true" ]; then
-  usage
-  exit 1
-fi
-
-echo_bold() {
-  printf "\\033[1m%s\\033[0m\\n" "${@}"
-}
-
-echo_bold "==> Creating new configuration at: $CONFIG_DIR"
-mkdir -p "$CONFIG_DIR" "$CERTS_DIR"
-
-echo_bold "==> Bootstrapping easy-rsa..."
-source "${DIR}/ssl-common.sh"
-
-echo_bold "==> Generating root CA cert..."
-# shellcheck source=scripts/gen-root-ca
-source "${DIR}/gen-root-ca" "${DOMAIN}" "${CERTS_DIR}"
-
-echo_bold "==> Generating root cert chain for haproxy..."
-# shellcheck source=scripts/gen-root-cert
-source "${DIR}/gen-root-cert" "${DOMAIN}" "${CERTS_DIR}"
-
-echo_bold "==> Generating token auth cert..."
-# shellcheck source=scripts/gen-token-auth-cert
-source "${DIR}/gen-token-auth-cert" "${DOMAIN}" "${CERTS_DIR}"
-
-echo_bold "==> Generating VPN CA, cert and dhparam (this may take a while)..."
-# shellcheck source=scripts/gen-vpn-certs
-source "${DIR}/gen-vpn-certs" "${DOMAIN}" "${CERTS_DIR}"
-
-echo_bold "==> Setting up environment..."
-# shellcheck source=scripts/make-env
-cat >"${CONFIG_DIR}/activate" <(source "${DIR}/make-env")
-
-echo_bold "==> Adding default compose file..."
-cp "${BASE_DIR}/compose/template.yml" "${CONFIG_DIR}/docker-compose.yml"
-
-if [ "${patch_hosts}" = "true" ]; then
-  echo_bold "==> Patching /etc/hosts..."
-  # shellcheck source=scripts/patch-hosts
-  source "${DIR}/patch-hosts" "${DOMAIN}"
-fi
-
-echo_bold "==> Success!"
-echo '  - Start the instance with: ./scripts/compose up -d'
-echo '  - Stop the instance with: ./scripts/compose stop'
-echo '  - To create the superuser, see: ./scripts/create-superuser -h'
-echo "  - Use the following certificate with Balena CLI: ${CERTS_DIR}/root/ca.crt"

scripts/ssl-common.sh

@@ -1,25 +0,0 @@
-#!/bin/bash -e
-# shellcheck disable=SC2034
-
-# ensure we have `easyrsa` available
-if [ -z "${easyrsa_bin-}" ] || [ ! -x "${easyrsa_bin}" ]; then
-    easyrsa_bin="$(command -v easyrsa 2>/dev/null || true)"
-    if [ -z "${easyrsa_bin}" ]; then
-        easyrsa_dir="$(mktemp -dt easyrsa.XXXXXXXX)"
-        easyrsa_url="https://github.com/OpenVPN/easy-rsa/releases/download/v3.0.5/EasyRSA-nix-3.0.5.tgz"
-        echo "  - Downloading easy-rsa..."
-        (cd "${easyrsa_dir}"; curl -sL "${easyrsa_url}" | tar xz --strip-components=1)
-        easyrsa_bin="${easyrsa_dir}/easyrsa"
-        # shellcheck disable=SC2064
-        trap "rm -rf \"${easyrsa_dir}\"" EXIT
-    fi
-    export EASYRSA_BATCH=1
-    export EASYRSA_KEY_SIZE=4096
-fi
-
-# setup ROOT_PKI path
-ROOT_PKI="$(realpath "${OUT}/root")"
-
-# global expiry settings
-CA_EXPIRY_DAYS=3650
-CRT_EXPIRY_DAYS=730

src/README

@@ -1 +0,0 @@
-This is the working folder for any specific container you might want to work on.

src/balena-tests/Dockerfile

@@ -0,0 +1,34 @@
+FROM ubuntu:22.04
+
+# renovate: datasource=github-releases depName=balena-io/balena-cli
+ARG BALENA_CLI_VERSION=v18.2.2
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+	bash \
+	ca-certificates \
+	curl \
+	jq \
+	openssl \
+	procmail \
+	qemu-utils \
+	unzip \
+	wget \
+    && rm -rf /var/lib/apt/lists/*
+
+# https://docs.docker.com/engine/install/ubuntu/#install-using-the-convenience-script
+RUN curl -fsSL https://get.docker.com | sh
+
+WORKDIR /opt
+
+RUN set -x; arch=$(uname -m | sed 's/86_64/64/g') \
+    && wget -q "https://github.com/balena-io/balena-cli/releases/download/${BALENA_CLI_VERSION}/balena-cli-${BALENA_CLI_VERSION}-linux-${arch}-standalone.zip" \
+    && unzip -q "balena-cli-${BALENA_CLI_VERSION}-linux-${arch}-standalone.zip" \
+    && rm -rf "balena-cli-${BALENA_CLI_VERSION}-linux-${arch}-standalone.zip"
+
+ENV PATH=/opt/balena-cli:${PATH}
+
+COPY functions balena.sh /usr/sbin/
+
+WORKDIR /balena
+
+CMD /usr/sbin/balena.sh

src/balena-tests/balena.sh

@@ -0,0 +1,328 @@
+#!/usr/bin/env bash
+
+# shellcheck disable=SC2154,SC2034,SC1090
+set -ae
+
+curl_opts="--retry 3 --fail"
+if [[ $VERBOSE =~ on|On|Yes|yes|true|True ]]; then
+    set -x
+    curl_opts="${curl_opts} --verbose"
+else
+    curl_opts="${curl_opts} --silent"
+fi
+
+source /usr/sbin/functions
+
+function remove_test_assets() {
+    rm -rf /balena/config.json \
+      "${GUEST_IMAGE}" \
+      "${GUEST_IMAGE%.*}.ready" \
+      "${tmpbuild}" \
+      /tmp/*.img
+}
+
+function remove_update_lock() {
+    rm -f /tmp/balena/updates.lock
+}
+
+function cleanup() {
+   shutdown_dut
+   remove_test_assets
+   remove_update_lock
+
+   # crash loop backoff
+   sleep "$(( (RANDOM % 5) + 5 ))s"
+}
+trap 'cleanup' EXIT
+
+function shutdown_dut() {
+    local balena_device_uuid
+    balena_device_uuid="$(cat </balena/config.json | jq -r .uuid)"
+
+    if [[ -n $balena_device_uuid ]]; then
+        with_backoff balena device "${balena_device_uuid}"
+        balena device shutdown -f "${balena_device_uuid}" || true
+    fi
+}
+
+function set_update_lock {
+    if [[ -n "$BALENA_SUPERVISOR_ADDRESS" ]] && [[ -n "$BALENA_SUPERVISOR_API_KEY" ]]; then
+        while [[ $(curl ${curl_opts} "${BALENA_SUPERVISOR_ADDRESS}/v1/device?apikey=${BALENA_SUPERVISOR_API_KEY}" \
+          -H "Content-Type: application/json" | jq -r '.update_pending') == 'true' ]]; do
+
+            curl ${curl_opts} "${BALENA_SUPERVISOR_ADDRESS}/v1/device?apikey=${BALENA_SUPERVISOR_API_KEY}" \
+              -H "Content-Type: application/json" | jq -r
+
+            sleep "$(( (RANDOM % 3) + 3 ))s"
+        done
+        sleep "$(( (RANDOM % 5) + 5 ))s"
+
+        # https://www.balena.io/docs/learn/deploy/release-strategy/update-locking/
+        lockfile /tmp/balena/updates.lock
+    fi
+}
+
+function update_ca_certificates() {
+    # only set CA bundle if using private certificate chain
+    if [[ -e "${CERTS}/ca-bundle.pem" ]]; then
+        if [[ "$(readlink -f "${CERTS}/${TLD}-chain.pem")" =~ \/private\/ ]]; then
+            mkdir -p /usr/local/share/ca-certificates
+            cat <"${CERTS}/ca-bundle.pem" > /usr/local/share/ca-certificates/balenaRootCA.crt
+            # shellcheck disable=SC2034
+            CURL_CA_BUNDLE=${CURL_CA_BUNDLE:-${CERTS}/ca-bundle.pem}
+            NODE_EXTRA_CA_CERTS=${NODE_EXTRA_CA_CERTS:-${CURL_CA_BUNDLE}}
+            # (TBC) refactor to use NODE_EXTRA_CA_CERTS instead of ROOT_CA
+            # https://github.com/balena-io/e2e/blob/master/conf.js#L12-L14
+            # https://github.com/balena-io/e2e/blob/master/Dockerfile#L82-L83
+            # ... or
+            # https://thomas-leister.de/en/how-to-import-ca-root-certificate/
+            # https://github.com/puppeteer/puppeteer/issues/2377
+            ROOT_CA=${ROOT_CA:-$(cat <"${NODE_EXTRA_CA_CERTS}" | openssl base64 -A)}
+        else
+            rm -f /usr/local/share/ca-certificates/balenaRootCA.crt
+            unset NODE_EXTRA_CA_CERTS CURL_CA_BUNDLE ROOT_CA
+        fi
+        update-ca-certificates
+    fi
+}
+
+function wait_for_api() {
+    while ! curl ${curl_opts} "https://api.${DNS_TLD}/ping"; do
+        sleep "$(( (RANDOM % 5) + 5 ))s"
+    done
+}
+
+function open_balena_login() {
+    while ! balena login --credentials \
+      --email "${SUPERUSER_EMAIL}" \
+      --password "${SUPERUSER_PASSWORD}"; do
+        sleep "$(( (RANDOM % 5) + 5 ))s"
+    done
+}
+
+function create_fleet() {
+    if ! balena fleet "${TEST_FLEET}"; then
+        # wait for API to load DT contracts
+        while ! balena fleet create "${TEST_FLEET}" --type "${DEVICE_TYPE}"; do
+            sleep "$(( (RANDOM % 5) + 5 ))s"
+        done
+
+        # FIXME: on openBalena 'balena devices supported' always returns empty list
+        balena devices supported
+    fi
+}
+
+function download_os_image() {
+    if ! [[ -s "$GUEST_IMAGE" ]]; then
+        with_backoff wget -qO /tmp/balena.zip \
+          "${BALENA_API_URL}/download?deviceType=${DEVICE_TYPE}&version=${OS_VERSION:1}&fileType=.zip"
+
+        unzip -oq /tmp/balena.zip -d /tmp
+
+        cat <"$(find /tmp/ -type f -name '*.img' | head -n 1)" >"${GUEST_IMAGE}"
+
+        rm /tmp/balena.zip
+    fi
+}
+
+function configure_virtual_device() {
+    while ! [[ -s "$GUEST_IMAGE" ]]; do sleep "$(( (RANDOM % 5) + 5 ))s"; done
+
+    if ! [[ -s /balena/config.json ]]; then
+        balena_device_uuid="$(openssl rand -hex 16)"
+
+        with_backoff balena device register "${TEST_FLEET}" \
+          --uuid "${balena_device_uuid}"
+
+        with_backoff balena config generate \
+          --version "${OS_VERSION:1}" \
+          --device "${balena_device_uuid}" \
+          --network ethernet \
+          --appUpdatePollInterval 10 \
+          --dev \
+          --output /balena/config.json
+    fi
+    cat </balena/config.json | jq -re
+
+    with_backoff balena os configure "${GUEST_IMAGE}" \
+      --fleet "${TEST_FLEET}" \
+      --version "${OS_VERSION#v}" \
+      --config-network ethernet \
+      --config /balena/config.json
+
+    touch "${GUEST_IMAGE%.*}.ready"
+}
+
+function check_device_status() {
+    if [[ -e /balena/config.json ]]; then
+        balena_device_uuid="$(cat </balena/config.json | jq -r .uuid)"
+
+        if [[ -n $balena_device_uuid ]]; then
+            is_online="$(balena devices --json --fleet "${TEST_FLEET}" \
+              | jq -r --arg uuid "${balena_device_uuid}" '.[] | select(.uuid==$uuid).is_online == true')"
+
+            if [[ $is_online =~ true ]]; then
+                return 0
+            else
+                return 1
+            fi
+        fi
+    fi
+}
+
+function wait_for_device() {
+    while ! check_device_status; do sleep "$(( (RANDOM % 5) + 5 ))s"; done
+}
+
+function registry_auth() {
+    if [[ -n $REGISTRY_USER ]] && [[ -n $REGISTRY_PASS ]]; then
+        with_backoff docker login -u "${REGISTRY_USER}" -p "${REGISTRY_PASS}"
+
+        printf '{"https://index.docker.io/v1/": {"username":"%s", "password":"$s"}}' \
+          "${REGISTRY_USER}" "${REGISTRY_PASS}" | jq -r > ~/.balena/secrets.json
+    fi
+}
+
+function deploy_release() {
+    tmpbuild="$(mktemp -d)"
+    pushd "${tmpbuild}"
+
+    echo 'FROM hello-world' >Dockerfile
+
+    while ! balena deploy \
+      --ca "${DOCKER_CERT_PATH}/ca.pem" \
+      --cert "${DOCKER_CERT_PATH}/cert.pem" \
+      --key "${DOCKER_CERT_PATH}/key.pem" \
+      "${TEST_FLEET}"; do
+
+        sleep "$(( (RANDOM % 5) + 5 ))s"
+    done
+    popd
+}
+
+function get_releases() {
+      with_backoff balena releases --json "${TEST_FLEET}"
+}
+
+function get_release_commit() {
+      echo "$(get_releases)" | jq -re \
+        'select((.[].status=="success")
+        and (.[].is_invalidated==false)
+        and (.[].is_final==true)
+        and (.[].release_type=="final"))[0].commit'
+}
+
+function get_release_id() {
+      echo "$(get_releases)" | jq -re \
+        'select((.[].status=="success")
+        and (.[].is_invalidated==false)
+        and (.[].is_final==true)
+        and (.[].release_type=="final"))[0].id'
+}
+
+function supervisor_update_target_state() {
+    local balena_device_uuid
+    balena_device_uuid="$(cat </balena/config.json | jq -r .uuid)"
+
+    if [[ -n $balena_device_uuid ]]; then
+        while ! curl ${curl_opts} "https://api.${DNS_TLD}/supervisor/v1/update" \
+          --header "Content-Type: application/json" \
+          --header "Authorization: Bearer $(cat <~/.balena/token)" \
+          --data "{\"uuid\": \"${balena_device_uuid}\", \"data\": {\"force\": true}}"; do
+
+            sleep "$(( (RANDOM % 5) + 5 ))s"
+        done
+    fi
+}
+
+function check_running_release() {
+    local balena_device_uuid
+    balena_device_uuid="$(cat </balena/config.json | jq -r .uuid)"
+
+    local should_be_running_release
+    should_be_running_release="$(get_release_commit)"
+    [[ -z $should_be_running_release ]] && false
+
+    if [[ -n $balena_device_uuid ]]; then
+        while ! [[ $(balena device "${balena_device_uuid}" | grep -E ^COMMIT | awk '{print $2}') =~ ${should_be_running_release} ]]; do
+            running_release_id="$(balena device "${balena_device_uuid}" | grep -E ^COMMIT | awk '{print $2}')"
+            printf 'please wait, device %s should be running %s, but is still running %s...\n' \
+              "${balena_device_uuid}" \
+              "${1}" \
+              "${running_release_id}"
+
+            sleep "$(( (RANDOM % 5) + 5 ))s"
+        done
+    fi
+}
+
+function get_os_version() {
+    local BALENARC_BALENA_URL
+    BALENARC_BALENA_URL="$(echo "${BALENA_API_URL}" | sed 's#https://api\.##g')"
+
+    local os_version
+    os_version=${OS_VERSION:-$(with_backoff balena os versions "${DEVICE_TYPE}" | head -n 1)}
+    echo "${os_version}"
+}
+
+function upload_release_asset() {
+    if [[ "$RELEASE_ASSETS_TEST" =~ true ]]; then
+        local release_id
+        release_id=${1:-1}
+        release_asset="$(find / -type f -name '*.png' | head -n 1)"
+
+        curl ${curl_opts} "https://api.${DNS_TLD}/resin/release_asset" \
+          --header "Authorization: Bearer $(cat <~/.balena/token)" \
+          --form "asset=@${release_asset}" \
+          --form "release=${release_id}" \
+          --form "asset_key=$((RANDOM))-$(basename ${release_asset})" \
+          | jq -re .asset.href \
+          | xargs curl ${curl_opts} -o "/tmp/$((RANDOM))-$(basename ${release_asset})"
+    fi
+}
+
+# --- main
+if [[ "$PRODUCTION_MODE" =~ true ]]; then
+    exit
+fi
+
+if [[ -n "${BALENA_DEVICE_UUID}" ]]; then
+    # prepend the device UUID if running on balenaOS
+    TLD="${BALENA_DEVICE_UUID}.${DNS_TLD}"
+else
+    TLD="${DNS_TLD}"
+fi
+
+BALENA_API_URL=${BALENA_API_URL:-https://api.balena-cloud.com}
+BALENARC_BALENA_URL="${DNS_TLD}"
+CERTS=${CERTS:-/certs}
+CONF=${CONF:-/balena/${TLD}.env}
+DEVICE_TYPE=${DEVICE_TYPE:-generic-amd64}
+GUEST_DISK_SIZE=${GUEST_DISK_SIZE:-8}
+GUEST_IMAGE=${GUEST_IMAGE:-/balena/balena.img}
+OS_VERSION="$(get_os_version)"
+TEST_FLEET=${TEST_FLEET:-test-fleet}
+
+[[ -f "$CONF" ]] && source "${CONF}"
+
+update_ca_certificates  # ensure self-signed root CA certificate(s) trust
+
+registry_auth  # optionally authenticate with DockerHub (rate-limiting)
+
+wait_for_api  # spin here until the API is responding
+
+balena whoami || open_balena_login  # spin here until authenticated
+
+create_fleet  # spin here until the fleet is created
+
+# critical section
+set_update_lock
+download_os_image
+configure_virtual_device
+deploy_release
+upload_release_asset "$(get_release_id)"  # upload an additional asset to a release
+remove_update_lock
+# .. end
+
+wait_for_device  # spin here until test-device comes online
+check_running_release  # .. and ensure the device is running our release

src/balena-tests/functions

@@ -0,0 +1,32 @@
+# https://coderwall.com/p/--eiqg/exponential-backoff-in-bash
+function with_backoff() {
+    local max_attempts=${ATTEMPTS-5}
+    local timeout=${TIMEOUT-1}
+    local attempt=0
+    local exitCode=0
+
+    set +e
+    while [[ $attempt < $max_attempts ]]
+    do
+        "$@"
+        exitCode=$?
+
+        if [[ $exitCode == 0 ]]
+        then
+          break
+        fi
+
+        echo "Failure! Retrying in $timeout.." 1>&2
+        sleep "$timeout"
+        attempt=$(( attempt + 1 ))
+        timeout=$(( timeout * 2 ))
+    done
+
+    if [[ $exitCode != 0 ]]
+    then
+        echo "You've failed me for the last time! ($*)" 1>&2
+    fi
+
+    set -e
+    return $exitCode
+}

src/cert-manager/Dockerfile

@@ -0,0 +1,4 @@
+# https://github.com/balena-io/cert-manager
+FROM balena/cert-manager:v0.2.2
+
+COPY *.json /opt/

src/cert-manager/certs.json

@@ -0,0 +1,63 @@
+[
+  {
+    "request": {
+      "key": {
+        "algo": "${key_algo}",
+        "size": ${key_size}
+      },
+      "hosts": ${hosts},
+      "names": [
+        {
+          "C": "${country}",
+          "L": "${locality_name}",
+          "O": "${org}",
+          "OU": "${org_unit}",
+          "ST": "${state}"
+        }
+      ],
+      "CN": "${TLD}"
+    }
+  },
+  {
+    "request": {
+      "key": {
+        "algo": "${key_algo}",
+        "size": ${key_size}
+      },
+      "hosts": [
+        "vpn.${TLD}"
+      ],
+      "names": [
+        {
+          "C": "${country}",
+          "L": "${locality_name}",
+          "O": "${org}",
+          "OU": "${org_unit}",
+          "ST": "${state}"
+        }
+      ],
+      "CN": "vpn.${TLD}"
+    }
+  },
+  {
+	"request": {
+	  "key": {
+		"algo": "${key_algo}",
+		"size": ${key_size}
+	  },
+	  "hosts": [
+		"api.${TLD}"
+	  ],
+	  "names": [
+		{
+		  "C": "${country}",
+		  "L": "${locality_name}",
+		  "O": "${org}",
+		  "OU": "${org_unit}",
+		  "ST": "${state}"
+		}
+	  ],
+	  "CN": "api.${TLD}"
+	}
+  }
+]

src/cert-manager/keys.json

@@ -0,0 +1 @@
+[]

src/haproxy-sidecar/Dockerfile

@@ -0,0 +1,7 @@
+FROM docker
+
+COPY balena.sh /usr/local/bin/balena.sh
+
+RUN apk add --no-cache bash
+
+CMD /usr/local/bin/balena.sh

src/haproxy-sidecar/balena.sh

@@ -0,0 +1,75 @@
+#!/usr/bin/env bash
+
+set -ea
+
+[[ $VERBOSE =~ on|On|Yes|yes|true|True ]] && set -x
+
+function cleanup() {
+   rm -f /host/run/docker.sock
+
+   # crash loop backoff
+   sleep "$(( (RANDOM % 5) + 5 ))s"
+}
+
+trap 'cleanup' EXIT
+
+if [ -S /host/run/balena-engine.sock ]; then
+    ln -s /host/run/balena-engine.sock /host/run/docker.sock
+fi
+
+which curl || apk add curl --no-cache
+which jq || apk add jq --no-cache
+
+if docker inspect "${BALENA_APP_UUID}_default" --format "{{.ID}}"; then
+    network="${BALENA_APP_UUID}_default"
+elif docker inspect "${BALENA_APP_ID}_default" --format "{{.ID}}"; then
+    network="${BALENA_APP_ID}_default"
+else
+    network=open-balena_default
+fi
+
+# shellcheck disable=SC2153
+for alias in ${ALIASES//,/ }; do
+    hostname="${alias}.${DNS_TLD}"
+    aliases="--alias ${hostname} ${aliases}"
+done
+
+while true; do
+    if [[ -n $BALENA_SUPERVISOR_ADDRESS ]] && [[ -n $BALENA_SUPERVISOR_API_KEY ]]; then
+        while [[ "$(curl --silent --retry 3 --fail \
+          "${BALENA_SUPERVISOR_ADDRESS}/v1/device?apikey=${BALENA_SUPERVISOR_API_KEY}" \
+          -H "Content-Type:application/json" | jq -r '.update_pending')" =~ true ]]; do
+            sleep "$(( (RANDOM % 3) + 3 ))s"
+        done
+        sleep "$(( (RANDOM % 5) + 5 ))s"
+    fi
+
+    while [[ "$(docker ps \
+      --filter "name=haproxy" \
+      --filter "expose=1936/tcp" \
+      --filter "status=running" \
+      --filter "network=${network}" \
+      --format "{{.ID}}")" == '' ]]; do
+        sleep "$(( (RANDOM % 3) + 3 ))s"
+    done
+
+    haproxy="$(docker ps \
+      --filter "name=haproxy" \
+      --filter "expose=1936/tcp" \
+      --filter "status=running" \
+      --filter "network=${network}" \
+      --format "{{.ID}}")"
+
+    if ! [[ $restarted == "${haproxy}" ]]; then
+        docker network disconnect "${network}" "${haproxy}"
+
+        # shellcheck disable=SC2086
+        docker network connect --alias haproxy ${aliases} "${network}" "${haproxy}"
+
+        docker restart "${haproxy}"
+
+        restarted="${haproxy}"
+    fi
+
+    sleep "$(( (RANDOM % 15) + 15 ))s"
+done

src/haproxy/Dockerfile

@@ -0,0 +1,4 @@
+# https://github.com/balena-io/open-balena-haproxy
+FROM balena/open-balena-haproxy:v4.3.1
+
+COPY haproxy.cfg /usr/local/etc/haproxy/haproxy.cfg

src/haproxy/haproxy.cfg

@@ -0,0 +1,176 @@
+global
+	tune.ssl.default-dh-param 1024
+	# https://github.com/haproxytech/haproxy-lua-cors
+	lua-load /usr/local/etc/haproxy/cors.lua
+	# https://www.haproxy.com/blog/introduction-to-haproxy-logging/
+	log stdout format raw daemon "${LOGLEVEL}"
+	log stderr format raw daemon "${LOGLEVEL}"
+	ssl-default-bind-options ssl-min-ver TLSv1.2
+
+defaults
+	balance roundrobin
+	default-server init-addr last,libc,none
+	default-server inter 3s rise 2 fall 3
+	log global
+	mode http
+	option contstats
+	option dontlognull
+	option forwardfor
+	option httplog
+	timeout client 63s
+	timeout connect 5s
+	timeout http-keep-alive 1s
+	timeout http-request 63s
+	timeout server 63s
+	timeout tunnel 3600s
+
+resolvers docker-bridge-resolver
+	nameserver docker-resolver 127.0.0.11:53
+	hold valid 0ms
+
+http-errors balena-http-errors
+	errorfile 400 /etc/haproxy/errors/400.http
+	errorfile 401 /etc/haproxy/errors/401.http
+	errorfile 403 /etc/haproxy/errors/403.http
+	errorfile 404 /etc/haproxy/errors/404.http
+	errorfile 500 /etc/haproxy/errors/500.http
+	errorfile 502 /etc/haproxy/errors/502.http
+	errorfile 503 /etc/haproxy/errors/503.http
+
+userlist balena
+	user balena insecure-password "${BALENA_DEVICE_UUID}"
+
+listen haproxy-stats
+	bind :::1936 v4v6 ssl crt "${CERT_CHAIN_PATH}" alpn h2,http/1.1
+	stats auth "balena:${BALENA_DEVICE_UUID}"
+	stats enable
+	stats uri /metrics
+
+frontend http
+	bind :::80 v4v6
+	default_backend api-backend
+	errorfiles balena-http-errors
+	http-request capture req.hdr(Host) len 15
+	http-response lua.cors
+	# https://www.haproxy.com/blog/haproxy-log-customization/
+	log-format "%ci:%cp [%tr] %ft %b/%s %TR/%Tw/%Tc/%Tr/%Ta %ST %B %CC %CS %tsc %ac/%fc/%bc/%sc/%rc %sq/%bq %hr %hs %{+Q}r"
+
+	acl api_dead nbsrv(api-backend) lt 1
+	acl registry_dead nbsrv(registry-backend) lt 1
+	acl vpn_dead nbsrv(vpn-backend) lt 1
+	monitor-uri /health
+	monitor fail if api_dead registry_dead vpn_dead
+
+	acl host-api-backend hdr_beg(host) -i "api."
+	# default public device URL(s) always go to the API
+	acl host-pdu-default hdr(host) -m reg -i "\.?([0-9a-f]{32}|${BALENA_DEVICE_UUID})\.(devices|balena-?(.*)-devices)\."
+	use_backend api-backend if host-api-backend || host-pdu-default
+
+	acl host-registry-backend hdr_beg(host) -i "registry2."
+	http-request add-header X-Forwarded-Proto http if host-registry-backend
+	use_backend registry-backend if host-registry-backend
+
+	acl host-s3-backend hdr_beg(host) -i "s3."
+	http-request add-header X-Forwarded-Proto http if host-s3-backend
+	use_backend s3-backend if host-s3-backend
+
+	acl host-minio-backend hdr_beg(host) -i "minio."
+	http-request add-header X-Forwarded-Proto http if host-minio-backend
+	use_backend minio-backend if host-minio-backend
+
+# routes between OpenVPN, SSL and HTTPS traffic
+frontend tcp-router
+	mode tcp
+	option tcplog
+	log-format "%ci:%cp [%t] %ft %b/%s %Tw/%Tc/%Tt %B %ts %ac/%fc/%bc/%sc/%rc %sq/%bq"
+	bind :::443 v4v6
+	tcp-request inspect-delay 2s
+	tcp-request content accept if { req.ssl_hello_type 1 }
+	acl is_ssl req.ssl_ver 2:3.4
+
+	acl sni-host-tunnel req_ssl_sni -m beg "tunnel."
+	use_backend redirect-to-tunnel if sni-host-tunnel
+
+	# everything else => HTTPS
+	use_backend redirect-to-https if is_ssl
+
+	# or VPN
+	use_backend vpn-backend if !is_ssl
+
+backend redirect-to-tunnel
+	mode tcp
+	server localhost 127.0.0.1:3129 send-proxy-v2
+
+# https://stackoverflow.com/a/39213442/1559300
+listen tunnel-backend
+	mode tcp
+	option tcplog
+	log-format "%ci:%cp [%t] %ft %b/%s %Tw/%Tc/%Tt %B %ts %ac/%fc/%bc/%sc/%rc %sq/%bq"
+	bind 127.0.0.1:3129 ssl crt "${CERT_CHAIN_PATH}" alpn h2,http/1.1 accept-proxy
+	server tunnel vpn:3128 resolvers docker-bridge-resolver resolve-prefer ipv4 check port 3128
+
+backend redirect-to-https
+	mode tcp
+	server localhost 127.0.0.1:444 send-proxy-v2
+
+frontend https
+	bind 127.0.0.1:444 ssl crt "${CERT_CHAIN_PATH}" alpn h2,http/1.1 accept-proxy
+	default_backend api-backend
+	errorfiles balena-http-errors
+	http-request add-header X-Forwarded-Proto https
+	http-request capture req.hdr(Host) len 15
+	http-response lua.cors
+	log-format "%ci:%cp [%tr] %ft %b/%s %TR/%Tw/%Tc/%Tr/%Ta %ST %B %CC %CS %tsc %ac/%fc/%bc/%sc/%rc %sq/%bq %hr %hs %{+Q}r"
+
+	acl host-api-backend hdr_beg(host) -i "api."
+	use_backend api-backend if host-api-backend
+
+	acl host-registry-backend hdr_beg(host) -i "registry2."
+	use_backend registry-backend if host-registry-backend
+
+	acl host-s3-backend hdr_beg(host) -i "s3."
+	use_backend s3-backend if host-s3-backend
+
+	acl host-minio-backend hdr_beg(host) -i "minio."
+	use_backend minio-backend if host-minio-backend
+
+	acl host-ca-backend hdr_beg(host) -i "ca."
+	# only allow CRL requests unauthenticated, protect everything else
+	acl balena-ca-crl path -i -m beg /api/v1/cfssl/crl
+	acl balena-ca-auth http_auth(balena)
+	http-request auth realm balena-ca if host-ca-backend !balena-ca-auth !balena-ca-crl
+	use_backend ca-backend if host-ca-backend
+
+	acl host-ocsp-backend hdr_beg(host) -i "ocsp."
+	use_backend ocsp-backend if host-ocsp-backend
+
+backend api-backend
+	server api api:80 resolvers docker-bridge-resolver resolve-prefer ipv4 check port 80
+
+backend registry-backend
+	server registry registry:80 resolvers docker-bridge-resolver resolve-prefer ipv4 check port 80
+
+backend s3-backend
+	server s3 s3:80 resolvers docker-bridge-resolver resolve-prefer ipv4 check port 80
+
+# https://github.com/minio/console
+backend minio-backend
+	server s3-console s3:43697 resolvers docker-bridge-resolver resolve-prefer ipv4 check port 43697
+
+backend db-backend
+	mode tcp
+	server db db:5432 resolvers docker-bridge-resolver resolve-prefer ipv4 check port 5432
+
+backend redis-backend
+	mode tcp
+	server redis redis:6379 resolvers docker-bridge-resolver resolve-prefer ipv4 check port 6379
+
+backend ca-backend
+	server cfssl-ca balena-ca:8888 resolvers docker-bridge-resolver resolve-prefer ipv4 check port 8888
+
+backend ocsp-backend
+	server cfssl-ocsp balena-ca:8889 resolvers docker-bridge-resolver resolve-prefer ipv4 check port 8889
+
+backend vpn-backend
+	mode tcp
+	server openvpn vpn:443 resolvers docker-bridge-resolver resolve-prefer ipv4 send-proxy-v2 check-send-proxy check port 443

src/tag-sidecar/Dockerfile

@@ -0,0 +1,5 @@
+FROM bash:alpine3.19
+
+COPY balena.sh /usr/local/bin/balena.sh
+
+CMD /usr/local/bin/balena.sh

src/tag-sidecar/balena.sh

@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+
+set -ea
+
+[[ $VERBOSE =~ on|On|Yes|yes|true|True ]] && set -x
+
+[[ $ENABLED == 'true' ]] || exit
+
+curl_with_opts() {
+    curl --fail --silent --retry 3 --connect-timeout 3 --compressed "$@"
+}
+
+get_aws_meta() {
+    if [[ $1 =~ ^.*/$ ]]; then
+        for key in $(curl_with_opts "$1"); do
+            get_aws_meta "$1${key}"
+        done
+    else
+        echo "$(echo "$1" | cut -c41-);$(curl_with_opts "$1" | tr '\n' ',')"
+    fi
+}
+
+if [[ -n $BALENA_API_URL ]] && [[ -n $BALENA_DEVICE_UUID ]] && [[ -n $BALENA_API_KEY ]]; then
+	which curl || apk add curl --no-cache
+	which jq || apk add jq --no-cache
+
+	device_id="$(curl_with_opts \
+	  "${BALENA_API_URL}/v6/device?\$filter=uuid%20eq%20'${BALENA_DEVICE_UUID}'" \
+	  -H "Content-Type: application/json" \
+	  -H "Authorization: Bearer ${BALENA_API_KEY}" | jq -r .d[].id)"
+
+	for key in $(curl_with_opts http://169.254.169.254/latest/meta-data \
+	  | grep -Ev 'iam|metrics|identity-credentials|network|events'); do
+		for kv in $(get_aws_meta "http://169.254.169.254/latest/meta-data/${key}"); do
+			tag_key="$(echo "${kv}" | awk -F';' '{print $1}')"
+			value="$(echo "${kv}" | awk -F';' '{print $2}')"
+
+			curl_with_opts "${BALENA_API_URL}/v6/device_tag" \
+			  -H "Content-Type: application/json" \
+			  -H "Authorization: Bearer ${BALENA_API_KEY}" \
+			  --data "{\"device\":\"${device_id}\",\"tag_key\":\"${tag_key}\",\"value\":\"${value}\"}"
+		done
+	done
+fi

src/test-device/Dockerfile

@@ -0,0 +1,11 @@
+# https://hub.docker.com/r/qemux/qemu-docker
+# https://github.com/qemus/qemu-docker
+FROM qemux/qemu-docker:4.24
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+	minicom \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY balena.sh /usr/sbin/
+
+WORKDIR /balena

src/test-device/balena.sh

@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+
+set -ae
+
+[[ $VERBOSE =~ on|On|Yes|yes|true|True ]] && set -x
+
+function cleanup() {
+   rm -f "${tmpimg}"
+
+   # crash loop backoff
+   sleep "$(( (RANDOM % 5) + 5 ))s"
+}
+trap 'cleanup' EXIT
+
+if [[ "$PRODUCTION_MODE" =~ true ]]; then
+    exit
+fi
+
+until test -f "${GUEST_IMAGE%.*}.ready"; do sleep "$(( (RANDOM % 5) + 5 ))s"; done
+
+tmpimg="$(mktemp)"
+cat <"${GUEST_IMAGE}" >"${tmpimg}"
+
+exec /usr/bin/qemu-system-x86_64 \
+  -bios /usr/share/ovmf/OVMF.fd \
+  -chardev socket,id=serial0,path=/run/console.sock,server=on,wait=off \
+  -cpu max \
+  -device ahci,id=ahci \
+  -device ide-hd,drive=disk,bus=ahci.0 \
+  -device virtio-net-pci,netdev=n1 \
+  -drive file="${tmpimg}",media=disk,cache=none,format=raw,if=none,id=disk \
+  -m "${MEMORY}" \
+  -machine q35 \
+  -netdev "user,id=n1,dns=127.0.0.1,guestfwd=tcp:10.0.2.100:80-cmd:netcat haproxy 80,guestfwd=tcp:10.0.2.100:443-cmd:netcat haproxy 443" \
+  -nodefaults \
+  -nographic \
+  -serial chardev:serial0 \
+  -smp "${CPU}"