SSH (client) verbosity config

* help debug random failures during test execution

change-type: patch

.github/actionlint.yaml

@@ -0,0 +1,8 @@
+self-hosted-runner:
+  # Labels of self-hosted runner in array of strings.
+  labels:
+  - "distro:jammy"
+# Configuration variables in array of strings defined in your repository or
+# organization. `null` means disabling configuration variables check.
+# Empty array means no configuration variable is allowed.
+config-variables: null

.github/workflows/renovate.json

@@ -0,0 +1,15 @@
+{
+  "customManagers": [
+    {
+      "customType": "regex",
+      "fileMatch": [
+        ".*"
+      ],
+      "matchStrings": [
+        ".*amiFilter=(?<packageName>.*?)\n(.*currentImageName=(?<currentDigest>.*?)\n)?(.*\n)?.*?(?<depName>[a-zA-Z0-9-_:]*)[ ]*?[:|=][ ]*?[\"|']?(?<currentValue>ami-[a-z0-9]{17})[\"|']?.*"
+      ],
+      "datasourceTemplate": "aws-machine-image",
+      "versioningTemplate": "aws-machine-image"
+    }
+  ]
+}

.github/workflows/tests.yml

@@ -73,13 +73,13 @@ jobs:
       name: ${{ matrix.target }}
 
     steps:
-    - uses: actions/checkout@b80ff79f1755d06ba70441c368a6fe801f5f3a62
+    - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332
 
     # https://github.com/unfor19/install-aws-cli-action
     - name: Setup awscli
       uses: unfor19/install-aws-cli-action@e8b481e524a99f37fbd39fdc1dcb3341ab091367 # v1
 
-    - uses: aws-actions/configure-aws-credentials@61a110527dcc9ccef6c109117050c80a00bec898
+    - uses: aws-actions/configure-aws-credentials@febab93d024649c3429d6b4609fe0a64bd9803f3
       with:
         aws-region: ${{ vars.AWS_REGION || 'us-east-1' }}
         role-session-name: github-${{ github.job }}-${{ github.run_id }}-${{ github.run_attempt }}
@@ -90,15 +90,19 @@ jobs:
     - name: install session-manager-plugin
       if: matrix.target == 'compose-private-pki'
       run: |
+        # shellcheck disable=SC2153
         runner_arch="$(echo "${RUNNER_ARCH}" | tr '[:upper:]' '[:lower:]' | sed 's/x64/64bit/g')"
 
-        session-manager-plugin || (curl -sSfo session-manager-plugin.deb https://s3.amazonaws.com/session-manager-downloads/plugin/latest/ubuntu_${runner_arch}/session-manager-plugin.deb \
+        session-manager-plugin || (curl -sSfo session-manager-plugin.deb "https://s3.amazonaws.com/session-manager-downloads/plugin/latest/ubuntu_${runner_arch}/session-manager-plugin.deb" \
           && sudo dpkg -i session-manager-plugin.deb \
           && rm -f session-manager-plugin.deb)
 
     # https://github.com/balena-io-examples/setup-balena-action
     - name: Setup balena CLI
       uses: balena-io-examples/setup-balena-action@main
+      with:
+        # renovate: datasource=github-releases depName=balena-io/balena-cli
+        cli-version: v18.2.17
 
     # https://github.com/pdcastro/ssh-uuid#why
     # https://github.com/pdcastro/ssh-uuid#linux-debian-ubuntu-others
@@ -144,7 +148,7 @@ jobs:
         set ${verbose}
 
         key_name="${{ matrix.target }}-${GITHUB_RUN_ID}-${GITHUB_RUN_NUMBER}-${GITHUB_RUN_ATTEMPT}"
-        echo "key_name=${key_name}" >> $GITHUB_OUTPUT
+        echo "key_name=${key_name}" >>"${GITHUB_OUTPUT}"
 
         set +x
         private_key_material="$(aws ec2 create-key-pair \
@@ -160,14 +164,13 @@ jobs:
         done <<< "${private_key_material}"
 
         ssh_private_key="$(cat << EOF
-        $(echo "${private_key_material}")
+        ${private_key_material}
         EOF
         )"
-        echo "ssh_private_key<<EOF" >> $GITHUB_OUTPUT
+        echo "ssh_private_key<<EOF" >>"${GITHUB_OUTPUT}"
         set ${verbose}
 
-        echo "${ssh_private_key}" >> $GITHUB_OUTPUT
-        echo "EOF" >> $GITHUB_OUTPUT
+        { echo "${ssh_private_key}"; echo "EOF"; } >>"${GITHUB_OUTPUT}"
 
         echo "ssh_public_key=${public_key}" >> "${GITHUB_OUTPUT}"
 
@@ -201,6 +204,7 @@ jobs:
         # the actual version deployed depends on the AWS EC2/AMI, defined in AWS_EC2_LAUNCH_TEMPLATE
         os_version="$(balena os versions ${{ vars.DEVICE_TYPE || 'generic-amd64' }} | head -n 1)"
 
+        # shellcheck disable=SC2046
         balena config generate \
           --version "${os_version}" \
           --device "${balena_device_uuid}" \
@@ -217,7 +221,7 @@ jobs:
           GITHUB_SHA GITHUB_WORKFLOW RUNNER_ARCH RUNNER_NAME RUNNER_OS)
 
         for github_var in "${github_vars[@]}"; do
-            balena tag set ${github_var} "${!github_var}" --device "${balena_device_uuid}"
+            balena tag set "${github_var}" "${!github_var}" --device "${balena_device_uuid}"
         done
 
         echo "balena_device_uuid=${balena_device_uuid}" >> "${GITHUB_OUTPUT}"
@@ -391,6 +395,7 @@ jobs:
 
         source src/balena-tests/functions
 
+        # shellcheck disable=SC2046,SC2043
         for subnet_id in ${{ env.AWS_VPC_SUBNET_IDS }}; do
             # spot, on-demand
             for market_type in ${{ vars.MARKET_TYPES || 'spot' }}; do
@@ -439,6 +444,12 @@ jobs:
 
         source src/balena-tests/functions
 
+        function cleanup() {
+            aws ec2 describe-instances --instance-ids ${{ steps.balena-sut.outputs.instance_id }} \
+              | jq -r .Reservations[].Instances[].StateReason
+        }
+        trap 'cleanup' EXIT
+
         with_backoff balena login --token '${{ secrets.BALENA_API_KEY }}'
 
         if ! [[ -e "${HOME}/.ssh/id_rsa" ]]; then
@@ -450,7 +461,7 @@ jobs:
 
         match=''
         for key in $(balena keys | grep -v ID | awk '{print $1}'); do
-            fp=$(balena key ${key} | tail -n 1 | ssh-keygen -E md5 -lf /dev/stdin | awk '{print $2}')
+            fp=$(balena key "${key}" | tail -n 1 | ssh-keygen -E md5 -lf /dev/stdin | awk '{print $2}')
             if [[ $fp =~ $(ssh-keygen -E md5 -lf "${HOME}/.ssh/id_rsa" | awk '{print $2}') ]]; then
                 match="${key}"
                 break
@@ -470,8 +481,8 @@ jobs:
             echo "::warning::Still working..."
             sleep "$(( (RANDOM % 5) + 5 ))s"
 
-            aws ec2 wait instance-running --instance-ids ${{ steps.balena-sut.outputs.instance_id }}
-            aws ec2 wait instance-status-ok --instance-ids ${{ steps.balena-sut.outputs.instance_id }}
+            aws ec2 wait instance-running --instance-ids ${{ steps.balena-sut.outputs.instance_id }} || exit 1
+            aws ec2 wait instance-status-ok --instance-ids ${{ steps.balena-sut.outputs.instance_id }} || exit 1
         done
 
         echo "key_id=${GITHUB_SHA}" >> "${GITHUB_OUTPUT}"
@@ -486,6 +497,12 @@ jobs:
 
         source src/balena-tests/functions
 
+        function cleanup() {
+            aws ec2 describe-instances --instance-ids ${{ steps.balena-sut.outputs.instance_id }} \
+              | jq -r .Reservations[].Instances[].StateReason
+        }
+        trap 'cleanup' EXIT
+
         with_backoff balena login --token '${{ secrets.BALENA_API_KEY }}'
 
         balena whoami && ssh-add -l
@@ -498,6 +515,9 @@ jobs:
           --compressed | jq -r '.update_pending')" =~ ^true$ ]]; do
 
             sleep "$(( ( RANDOM % ${{ env.RETRY }} ) + ${{ env.RETRY }} ))s"
+
+            aws ec2 wait instance-running --instance-ids ${{ steps.balena-sut.outputs.instance_id }} || exit 1
+            aws ec2 wait instance-status-ok --instance-ids ${{ steps.balena-sut.outputs.instance_id }} || exit 1
         done
 
         # wait for services to start running
@@ -509,8 +529,8 @@ jobs:
             echo "::warning::Still working..."
             sleep "$(( (RANDOM % 30) + 30 ))s"
 
-            aws ec2 wait instance-running --instance-ids ${{ steps.balena-sut.outputs.instance_id }}
-            aws ec2 wait instance-status-ok --instance-ids ${{ steps.balena-sut.outputs.instance_id }}
+            aws ec2 wait instance-running --instance-ids ${{ steps.balena-sut.outputs.instance_id }} || exit 1
+            aws ec2 wait instance-status-ok --instance-ids ${{ steps.balena-sut.outputs.instance_id }} || exit 1
         done
 
         # wait for Docker healthchecks
@@ -523,8 +543,8 @@ jobs:
             echo "::warning::Still working..."
             sleep "$(( (RANDOM % 30) + 30 ))s"
 
-            aws ec2 wait instance-running --instance-ids ${{ steps.balena-sut.outputs.instance_id }}
-            aws ec2 wait instance-status-ok --instance-ids ${{ steps.balena-sut.outputs.instance_id }}
+            aws ec2 wait instance-running --instance-ids ${{ steps.balena-sut.outputs.instance_id }} || exit 1
+            aws ec2 wait instance-status-ok --instance-ids ${{ steps.balena-sut.outputs.instance_id }} || exit 1
         done
 
     # (TBC) https://www.balena.io/docs/reference/supervisor/docker-compose/
@@ -541,6 +561,12 @@ jobs:
 
         source src/balena-tests/functions
 
+        function cleanup() {
+            aws ec2 describe-instances --instance-ids ${{ steps.balena-sut.outputs.instance_id }} \
+              | jq -r .Reservations[].Instances[].StateReason
+        }
+        trap 'cleanup' EXIT
+
         with_backoff balena login --token '${{ secrets.BALENA_API_KEY }}'
 
         balena whoami && ssh-add -l
@@ -563,8 +589,8 @@ jobs:
             echo "::warning::Still working..."
             sleep "$(( (RANDOM % 30) + 30 ))s"
 
-            aws ec2 wait instance-running --instance-ids ${{ steps.balena-sut.outputs.instance_id }}
-            aws ec2 wait instance-status-ok --instance-ids ${{ steps.balena-sut.outputs.instance_id }}
+            aws ec2 wait instance-running --instance-ids ${{ steps.balena-sut.outputs.instance_id }} || exit 1
+            aws ec2 wait instance-status-ok --instance-ids ${{ steps.balena-sut.outputs.instance_id }} || exit 1
         done
 
     - name: SUT&DUT (balena)
@@ -580,6 +606,12 @@ jobs:
 
         source src/balena-tests/functions
 
+        function cleanup() {
+            aws ec2 describe-instances --instance-ids ${{ steps.balena-sut.outputs.instance_id }} \
+              | jq -r .Reservations[].Instances[].StateReason
+        }
+        trap 'cleanup' EXIT
+
         with_backoff balena login --token '${{ secrets.BALENA_API_KEY }}'
 
         balena whoami && ssh-add -l
@@ -603,8 +635,8 @@ jobs:
             echo "::warning::Still working..."
             sleep "$(( ( RANDOM % ${{ env.RETRY }} ) + ${{ env.RETRY }} ))s"
 
-            aws ec2 wait instance-running --instance-ids ${{ steps.balena-sut.outputs.instance_id }}
-            aws ec2 wait instance-status-ok --instance-ids ${{ steps.balena-sut.outputs.instance_id }}
+            aws ec2 wait instance-running --instance-ids ${{ steps.balena-sut.outputs.instance_id }} || exit 1
+            aws ec2 wait instance-status-ok --instance-ids ${{ steps.balena-sut.outputs.instance_id }} || exit 1
         done
 
         # .. once the service exits with status == exited, it is assumed to be finished
@@ -620,8 +652,8 @@ jobs:
 
             sleep "$(( ( RANDOM % ${{ env.RETRY }} ) + ${{ env.RETRY }} ))s"
 
-            aws ec2 wait instance-running --instance-ids ${{ steps.balena-sut.outputs.instance_id }}
-            aws ec2 wait instance-status-ok --instance-ids ${{ steps.balena-sut.outputs.instance_id }}
+            aws ec2 wait instance-running --instance-ids ${{ steps.balena-sut.outputs.instance_id }} || exit 1
+            aws ec2 wait instance-status-ok --instance-ids ${{ steps.balena-sut.outputs.instance_id }} || exit 1
         done
 
         # .. check its exit code
@@ -774,6 +806,7 @@ jobs:
 
         cloud-init schema -c user-data.yml
 
+        # shellcheck disable=SC2046,SC2043
         for subnet_id in ${{ env.AWS_VPC_SUBNET_IDS }}; do
             # spot, on-demand
             for market_type in ${{ vars.MARKET_TYPES || 'spot' }}; do
@@ -783,7 +816,7 @@ jobs:
                       $([[ -n '${{ matrix.ami }}' ]] && echo '--image-id ${{ matrix.ami }}') \
                       --launch-template 'LaunchTemplateId=${{ env.AWS_EC2_LAUNCH_TEMPLATE }},Version=${{ matrix.launch_template_version }}' \
                       --instance-type "${instance_type}" \
-                      $([[ $market_type =~ spot ]] && echo '--instance-market-options MarketType=spot') \
+                      $([[ "$market_type" =~ spot ]] && echo '--instance-market-options MarketType=spot') \
                       --security-group-ids '${{ env.AWS_VPC_SECURITY_GROUP_IDS }}' \
                       --subnet-id "${subnet_id}" \
                       --key-name '${{ steps.generate-key-pair.outputs.key_name }}' \
@@ -801,7 +834,7 @@ jobs:
         [[ -z $response ]] && exit 1
 
         instance_id="$(echo "${response}" | jq -r '.Instances[].InstanceId')"
-        echo "instance_id=${instance_id}" >> $GITHUB_OUTPUT
+        echo "instance_id=${instance_id}" >>"${GITHUB_OUTPUT}"
 
         aws ec2 wait instance-running --instance-ids "${instance_id}"
         with_backoff aws ec2 wait instance-status-ok --instance-ids "${instance_id}"
@@ -836,6 +869,9 @@ jobs:
             aws logs put-retention-policy \
               --log-group-name open-balena-tests \
               --retention-in-days "${{ env.AWS_LOGS_RETENTION }}" || true
+
+            aws ec2 describe-instances --instance-ids ${{ steps.ubuntu-sut.outputs.instance_id }} \
+              | jq -r .Reservations[].Instances[].StateReason
         }
         trap 'log_output' EXIT
 
@@ -845,6 +881,8 @@ jobs:
             StrictHostKeyChecking no
             UserKnownHostsFile /dev/null
             ProxyCommand sh -c "aws ssm start-session --target %h --document-name AWS-StartSSHSession --parameters 'portNumber=%p'"
+            # QUIET, FATAL, ERROR, INFO, VERBOSE, DEBUG, DEBUG1, DEBUG2, and DEBUG3
+            LogLevel ${{ vars.SSH_VERBOSE || 'ERROR' }}     
         EOF
 
         # docs/getting-started.md
@@ -876,7 +914,7 @@ jobs:
         echo "${result}" | jq -re
         cid="$(echo "${result}" | jq -r .Command.CommandId)"
         iid="$(echo "${result}" | jq -r .Command.InstanceIds[0])"
-        ([[ -n "$cid" ]] && [[ -n "$iid" ]]) || false
+        { [[ -n "$cid" ]] && [[ -n "$iid" ]]; } || false
 
         # https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#grouping-log-lines
         CYAN='\033[0;36m'; NC='\033[0m'; echo -e "::group::${CYAN}open-balena-tests${NC}"
@@ -891,6 +929,7 @@ jobs:
           --log-group-name open-balena-tests \
           --log-stream-name-prefix "${cid}" | jq -r '.logStreams|length') -le 0 ]]; do
             echo '::info::waiting for logs...'
+            sleep $(((RANDOM%5) + 5))s
         done
         echo '::info::logs started'
 
@@ -899,26 +938,26 @@ jobs:
             with_backoff docker compose ls
             sleep $(((RANDOM%5) + 5))s
 
-            aws ec2 wait instance-running --instance-ids ${{ steps.ubuntu-sut.outputs.instance_id }}
-            aws ec2 wait instance-status-ok --instance-ids ${{ steps.ubuntu-sut.outputs.instance_id }}
+            aws ec2 wait instance-running --instance-ids ${{ steps.ubuntu-sut.outputs.instance_id }} || exit 1
+            aws ec2 wait instance-status-ok --instance-ids ${{ steps.ubuntu-sut.outputs.instance_id }} || exit 1
         done
         echo '::info::composition started'
 
         touch .env
         for service in sut dut; do
-        until [[ "$(docker compose ps --services "${service}" --status running)" =~ "${service}" ]]; do
+        until [[ "$(docker compose ps --services "${service}" --status running)" =~ ${service} ]]; do
             echo "::info::waiting for ${service}..."
             with_backoff docker compose ps
             sleep $(((RANDOM%5) + 5))s
 
-            aws ec2 wait instance-running --instance-ids ${{ steps.ubuntu-sut.outputs.instance_id }}
-            aws ec2 wait instance-status-ok --instance-ids ${{ steps.ubuntu-sut.outputs.instance_id }}
+            aws ec2 wait instance-running --instance-ids ${{ steps.ubuntu-sut.outputs.instance_id }} || exit 1
+            aws ec2 wait instance-status-ok --instance-ids ${{ steps.ubuntu-sut.outputs.instance_id }} || exit 1
         done
         echo "::info::${service} started"
         done
 
         echo '::info::settling down...'
-        sleep $(((RANDOM%30) + 15))s
+        sleep $(((RANDOM%15) + 15))s
 
         while [[ "$(aws ssm list-command-invocations --command-id "${cid}" \
           | jq -re '.CommandInvocations[].Status')" =~ InProgress ]]; do
@@ -927,8 +966,8 @@ jobs:
             echo '::info::still running...'
             sleep $(((RANDOM%1) + 1))s
 
-            aws ec2 wait instance-running --instance-ids ${{ steps.ubuntu-sut.outputs.instance_id }}
-            aws ec2 wait instance-status-ok --instance-ids ${{ steps.ubuntu-sut.outputs.instance_id }}
+            aws ec2 wait instance-running --instance-ids ${{ steps.ubuntu-sut.outputs.instance_id }} || exit 1
+            aws ec2 wait instance-status-ok --instance-ids ${{ steps.ubuntu-sut.outputs.instance_id }} || exit 1
         done
 
         aws ssm wait command-executed --command-id "${cid}" --instance-id "${iid}"
@@ -980,7 +1019,7 @@ jobs:
 
     # always destroy test EC2 instances even if the workflow is cancelled
     - name: destroy AWS test device(s)
-      if: always()
+      if: always() && matrix.target == 'balena-public-pki'
       run: |
         set -ue
 
@@ -993,33 +1032,55 @@ jobs:
               --instance-ids ${{ steps.balena-sut.outputs.instance_id }}
         fi
 
+      env:
+        AWS_DEFAULT_REGION: ${{ vars.AWS_REGION || 'us-east-1' }}
+
+    # always destroy test EC2 instances even if the workflow is cancelled
+    - name: destroy AWS test device(s)
+      if: always() && matrix.target == 'compose-private-pki'
+      run: |
+        set -ue
+
+        [[ '${{ vars.VERBOSE }}' =~ on|On|Yes|yes|true|True ]] && set -x
+
+        source src/balena-tests/functions
+
         if [[ -n '${{ steps.ubuntu-sut.outputs.instance_id }}' ]]; then
             with_backoff aws ec2 terminate-instances \
               --instance-ids ${{ steps.ubuntu-sut.outputs.instance_id }}
         fi
 
-        with_backoff aws ec2 describe-instances --filters Name=tag:GITHUB_SHA,Values=${GITHUB_SHA}-tests \
-          | jq -r .Reservations[].Instances[].InstanceId \
-          | xargs --no-run-if-empty aws ec2 terminate-instances --instance-ids
+      env:
+        AWS_DEFAULT_REGION: ${{ vars.AWS_REGION || 'us-east-1' }}
+
+    # always destroy stale test EC2 instances
+    - name: destroy stale AWS test device(s)
+      if: always()
+      run: |
+        set -ue
+
+        [[ '${{ vars.VERBOSE }}' =~ on|On|Yes|yes|true|True ]] && set -x
+
+        source src/balena-tests/functions
 
         stale_instances=$(mktemp)
         aws ec2 describe-instances --filters \
           Name=tag:Name,Values=open-balena-tests \
           Name=instance-state-name,Values=running \
-          | jq -re '.Reservations[].Instances[].InstanceId + " " + .Reservations[].Instances[].LaunchTime' > ${stale_instances} || true
+          | jq -re '.Reservations[].Instances[].InstanceId + " " + .Reservations[].Instances[].LaunchTime' > "${stale_instances}" || true
 
         if test -s "${stale_instances}"; then
             while IFS= read -r line; do
-                instance_id=$(echo ${line} | awk '{print $1}')
-                launch_time=$(echo ${line} | awk '{print $2}')
-                now=$(date +%s)
-                then=$(date --date ${launch_time} +%s)
-                days_since_launch=$(( (now - then) / 86400 ))
-                if [[ -n $days_since_launch ]] && [[ $days_since_launch -ge 1 ]]; then
-                    with_backoff aws ec2 terminate-instances --instance-ids ${instance_id}
+                instance_id="$(echo "${line}" | awk '{print $1}')"
+                launch_time="$(echo "${line}" | awk '{print $2}')"
+                now="$(date +%s)"
+                then="$(date --date "${launch_time}" +%s)"
+                days_since_launch="$(( (now - then) / 86400 ))"
+                if [[ -n "$days_since_launch" ]] && [[ $days_since_launch -ge 1 ]]; then
+                    with_backoff aws ec2 terminate-instances --instance-ids "${instance_id}"
                 fi
-            done <${stale_instances}
-            rm -f ${stale_instances}
+            done <"${stale_instances}"
+            rm -f "${stale_instances}"
         fi
 
       env:
@@ -1043,11 +1104,11 @@ jobs:
               "https://api.cloudflare.com/client/v4/zones?name=${{ matrix.dns_tld }}" \
               -H 'Authorization: Bearer ${{ secrets.CLOUDFLARE_API_TOKEN }}' | jq -r '.result[].id')"
 
-            for record in "$(curl --silent --retry ${{ env.RETRY }} \
+            for record in $(curl --silent --retry ${{ env.RETRY }} \
               "https://api.cloudflare.com/client/v4/zones/${zone_id}/dns_records" \
               -H 'Authorization: Bearer ${{ secrets.CLOUDFLARE_API_TOKEN }}' \
               | jq -r --arg match "${match}" '.result[] | select(((.type=="TXT") and (.name | contains($match))))' \
-              | base64)"; do
+              | base64); do
 
                 json="$(echo "${record}" | base64 -d | jq -r)"
                 id="$(echo "${json}" | jq -r .id)"

.versionbot/CHANGELOG.yml

@@ -1,3 +1,284 @@
+- commits:
+    - subject: Update balena/open-balena-s3 Docker tag to v2.28.48
+      hash: 7924ee920abca8caa0a499cd9a3af7efc6f06342
+      body: |
+        Update balena/open-balena-s3 from 2.28.47 to 2.28.48
+      footer:
+        Change-type: patch
+        change-type: patch
+      author: Self-hosted Renovate Bot
+      nested: []
+  version: 4.1.25
+  title: ""
+  date: 2024-07-09T16:57:27.685Z
+- commits:
+    - subject: Update balena/open-balena-api Docker tag to v25.1.29
+      hash: b3c8709b247dede72f9809d3109a80b82b207c84
+      body: |
+        Update balena/open-balena-api from 25.1.27 to 25.1.29
+      footer:
+        Change-type: patch
+        change-type: patch
+      author: Self-hosted Renovate Bot
+      nested: []
+  version: 4.1.24
+  title: ""
+  date: 2024-07-09T16:17:45.722Z
+- commits:
+    - subject: Update balena/open-balena-vpn Docker tag to v11.30.23
+      hash: 24ff05881929ed860a004f83fa3d65299db59361
+      body: |
+        Update balena/open-balena-vpn from 11.30.22 to 11.30.23
+      footer:
+        Change-type: patch
+        change-type: patch
+      author: Self-hosted Renovate Bot
+      nested: []
+  version: 4.1.23
+  title: ""
+  date: 2024-07-08T20:04:49.447Z
+- commits:
+    - subject: Update balena/open-balena-api Docker tag to v25.1.27
+      hash: 2a0edead1d2c5d41e98ed344efbb145dca8ec659
+      body: |
+        Update balena/open-balena-api from 25.1.24 to 25.1.27
+      footer:
+        Change-type: patch
+        change-type: patch
+      author: Self-hosted Renovate Bot
+      nested: []
+  version: 4.1.22
+  title: ""
+  date: 2024-07-08T16:37:30.112Z
+- commits:
+    - subject: Make tests more reliable
+      hash: b0f79a0d84742b0a96aaec934a2ad6ec72bb3d6e
+      body: |
+        * ensure config and creds are ready before attempting auth
+        * warn instead of error on DUT shutdown failures
+      footer:
+        change-type: patch
+      author: Anton Belodedenko
+      nested: []
+  version: 4.1.21
+  title: ""
+  date: 2024-07-08T14:49:09.165Z
+- commits:
+    - subject: Update dependency balena-io/balena-cli to v18.2.20
+      hash: d4683c028ae10fcfaf67c4f68dc322aa6602a896
+      body: |
+        Update balena-io/balena-cli from 18.2.19 to 18.2.20
+      footer:
+        Change-type: patch
+        change-type: patch
+      author: Self-hosted Renovate Bot
+      nested: []
+  version: 4.1.20
+  title: ""
+  date: 2024-07-05T22:14:20.547Z
+- commits:
+    - subject: Update dependency balena-io/balena-cli to v18.2.19
+      hash: dc7fd065f9265d6baf29e612a06d1e5109a47fa8
+      body: |
+        Update balena-io/balena-cli from 18.2.10 to 18.2.19
+      footer:
+        Change-type: patch
+        change-type: patch
+      author: Self-hosted Renovate Bot
+      nested: []
+  version: 4.1.19
+  title: ""
+  date: 2024-07-05T14:38:10.804Z
+- commits:
+    - subject: Update balena/open-balena-api Docker tag to v25.1.24
+      hash: cfb35f9d3946fd54e0057df5725114e56fd74296
+      body: |
+        Update balena/open-balena-api from 25.1.19 to 25.1.24
+      footer:
+        Change-type: patch
+        change-type: patch
+      author: Self-hosted Renovate Bot
+      nested: []
+  version: 4.1.18
+  title: ""
+  date: 2024-07-05T00:33:15.468Z
+- commits:
+    - subject: Update balena/open-balena-registry Docker tag to v2.39.60
+      hash: b023d3a47002823e18926b231aeadc104f4c0094
+      body: |
+        Update balena/open-balena-registry from 2.39.58 to 2.39.60
+      footer:
+        Change-type: patch
+        change-type: patch
+      author: Self-hosted Renovate Bot
+      nested: []
+  version: 4.1.17
+  title: ""
+  date: 2024-07-04T23:28:20.756Z
+- commits:
+    - subject: Update aws-actions/configure-aws-credentials digest to febab93
+      hash: 7fd045814c95f4566d38e53f7738dd67440f0115
+      body: |
+        Update aws-actions/configure-aws-credentials
+      footer:
+        Change-type: patch
+        change-type: patch
+      author: Self-hosted Renovate Bot
+      nested: []
+  version: 4.1.16
+  title: ""
+  date: 2024-07-04T22:17:01.050Z
+- commits:
+    - subject: pin balena-cli version
+      hash: 7ddb50c19fb470aa3cc975ce7a0afc2f6d47ba2f
+      body: ""
+      footer:
+        change-type: patch
+      author: Anton Belodedenko
+      nested: []
+  version: 4.1.15
+  title: ""
+  date: 2024-07-04T21:54:28.567Z
+- commits:
+    - subject: shellcheck/lint workflow(s)
+      hash: 0328960ecca2bd99cd5456392095e70227ccbcd0
+      body: ""
+      footer:
+        change-type: patch
+      author: Anton Belodedenko
+      nested: []
+  version: 4.1.14
+  title: ""
+  date: 2024-07-04T21:33:41.784Z
+- commits:
+    - subject: Update balena/open-balena-api Docker tag to v25.1.19
+      hash: 3e29f306972c9565574894d8a0d0a9d783d2df4b
+      body: |
+        Update balena/open-balena-api from 25.1.7 to 25.1.19
+      footer:
+        Change-type: patch
+        change-type: patch
+      author: Self-hosted Renovate Bot
+      nested: []
+  version: 4.1.13
+  title: ""
+  date: 2024-07-02T21:17:06.373Z
+- commits:
+    - subject: Only delete your (matrix) own test instance
+      hash: 96d56178897f96d470d23e313b5555412cfb7f4e
+      body: ""
+      footer:
+        change-type: patch
+      author: Anton Belodedenko
+      nested: []
+  version: 4.1.12
+  title: ""
+  date: 2024-07-02T20:48:44.727Z
+- commits:
+    - subject: Update balena/open-balena-s3 Docker tag to v2.28.47
+      hash: 52d0eb6c4d3fe244ca231ffe9507586145376f02
+      body: |
+        Update balena/open-balena-s3 from 2.28.45 to 2.28.47
+      footer:
+        Change-type: patch
+        change-type: patch
+      author: Self-hosted Renovate Bot
+      nested: []
+  version: 4.1.11
+  title: ""
+  date: 2024-07-02T16:08:15.757Z
+- commits:
+    - subject: Update aws-actions/configure-aws-credentials digest to af7e7d4
+      hash: 5c790da01dbb4dadb2827937a71f5e432e26208c
+      body: |
+        Update aws-actions/configure-aws-credentials
+      footer:
+        Change-type: patch
+        change-type: patch
+      author: Self-hosted Renovate Bot
+      nested: []
+  version: 4.1.10
+  title: ""
+  date: 2024-06-27T22:16:09.999Z
+- commits:
+    - subject: capture termination instance reason
+      hash: e54aae284624173d6e05b24f9dd0df7f13f0947e
+      body: ""
+      footer:
+        change-type: patch
+      author: Anton Belodedenko
+      nested: []
+  version: 4.1.9
+  title: ""
+  date: 2024-06-27T18:27:24.183Z
+- commits:
+    - subject: Exit instead of breaking out of while loops
+      hash: 6e4aae7a82aeedd57d3ace87de6702b8b267fada
+      body: ""
+      footer:
+        change-type: patch
+      author: Anton Belodedenko
+      nested: []
+  version: 4.1.8
+  title: ""
+  date: 2024-06-27T15:41:50.733Z
+- commits:
+    - subject: Renovate AWS/EC2 AMI(s)
+      hash: 5a3e2f1ac8cb4cd179c1d6b055e71ef447ddd227
+      body: ""
+      footer:
+        change-type: patch
+      author: Anton Belodedenko
+      nested: []
+  version: 4.1.7
+  title: ""
+  date: 2024-06-26T00:38:36.638Z
+- commits:
+    - subject: check if AWS/EC2 instance is still running
+      hash: 331c952a1f1235ff224cec6913d79e9bcc3e7cac
+      body: ""
+      footer:
+        change-type: patch
+      author: Anton Belodedenko
+      nested: []
+  version: 4.1.6
+  title: ""
+  date: 2024-06-25T23:43:44.670Z
+- commits:
+    - subject: break from wait loops if EC2 instance is terminated
+      hash: 85fd15d7b3528eb1f6dc0cb1955f5b9495cc4b02
+      body: ""
+      footer:
+        change-type: patch
+      author: Anton Belodedenko
+      nested: []
+  version: 4.1.5
+  title: ""
+  date: 2024-06-25T19:09:58.663Z
+- commits:
+    - subject: show EC2 instance termination status
+      hash: 538bcfe9593f3ad6fa406c820dfe71d92946c5f6
+      body: ""
+      footer:
+        change-type: patch
+      author: Anton Belodedenko
+      nested: []
+  version: 4.1.4
+  title: ""
+  date: 2024-06-25T18:27:13.520Z
+- commits:
+    - subject: Update actions/checkout digest to 692973e
+      hash: 422a34218c6955a5e65995961e3944b2c2f400b3
+      body: |
+        Update actions/checkout
+      footer:
+        Change-type: patch
+        change-type: patch
+      author: Self-hosted Renovate Bot
+      nested: []
+  version: 4.1.3
+  title: ""
+  date: 2024-06-25T14:16:47.810Z
 - commits:
     - subject: fail early on EC2 instance termination (i.e. spot capacity)
       hash: a2b62a6495e54aaa3983d0e4afe75612e053c7a7

CHANGELOG.md

@@ -4,6 +4,121 @@ All notable changes to this project will be documented in this file
 automatically by Versionist. DO NOT EDIT THIS FILE MANUALLY!
 This project adheres to [Semantic Versioning](http://semver.org/).
 
+# v4.1.25
+## (2024-07-09)
+
+* Update balena/open-balena-s3 Docker tag to v2.28.48 [Self-hosted Renovate Bot]
+
+# v4.1.24
+## (2024-07-09)
+
+* Update balena/open-balena-api Docker tag to v25.1.29 [Self-hosted Renovate Bot]
+
+# v4.1.23
+## (2024-07-08)
+
+* Update balena/open-balena-vpn Docker tag to v11.30.23 [Self-hosted Renovate Bot]
+
+# v4.1.22
+## (2024-07-08)
+
+* Update balena/open-balena-api Docker tag to v25.1.27 [Self-hosted Renovate Bot]
+
+# v4.1.21
+## (2024-07-08)
+
+* Make tests more reliable [Anton Belodedenko]
+
+# v4.1.20
+## (2024-07-05)
+
+* Update dependency balena-io/balena-cli to v18.2.20 [Self-hosted Renovate Bot]
+
+# v4.1.19
+## (2024-07-05)
+
+* Update dependency balena-io/balena-cli to v18.2.19 [Self-hosted Renovate Bot]
+
+# v4.1.18
+## (2024-07-05)
+
+* Update balena/open-balena-api Docker tag to v25.1.24 [Self-hosted Renovate Bot]
+
+# v4.1.17
+## (2024-07-04)
+
+* Update balena/open-balena-registry Docker tag to v2.39.60 [Self-hosted Renovate Bot]
+
+# v4.1.16
+## (2024-07-04)
+
+* Update aws-actions/configure-aws-credentials digest to febab93 [Self-hosted Renovate Bot]
+
+# v4.1.15
+## (2024-07-04)
+
+* pin balena-cli version [Anton Belodedenko]
+
+# v4.1.14
+## (2024-07-04)
+
+* shellcheck/lint workflow(s) [Anton Belodedenko]
+
+# v4.1.13
+## (2024-07-02)
+
+* Update balena/open-balena-api Docker tag to v25.1.19 [Self-hosted Renovate Bot]
+
+# v4.1.12
+## (2024-07-02)
+
+* Only delete your (matrix) own test instance [Anton Belodedenko]
+
+# v4.1.11
+## (2024-07-02)
+
+* Update balena/open-balena-s3 Docker tag to v2.28.47 [Self-hosted Renovate Bot]
+
+# v4.1.10
+## (2024-06-27)
+
+* Update aws-actions/configure-aws-credentials digest to af7e7d4 [Self-hosted Renovate Bot]
+
+# v4.1.9
+## (2024-06-27)
+
+* capture termination instance reason [Anton Belodedenko]
+
+# v4.1.8
+## (2024-06-27)
+
+* Exit instead of breaking out of while loops [Anton Belodedenko]
+
+# v4.1.7
+## (2024-06-26)
+
+* Renovate AWS/EC2 AMI(s) [Anton Belodedenko]
+
+# v4.1.6
+## (2024-06-25)
+
+* check if AWS/EC2 instance is still running [Anton Belodedenko]
+
+# v4.1.5
+## (2024-06-25)
+
+* break from wait loops if EC2 instance is terminated [Anton Belodedenko]
+
+# v4.1.4
+## (2024-06-25)
+
+* show EC2 instance termination status [Anton Belodedenko]
+
+# v4.1.3
+## (2024-06-25)
+
+* Update actions/checkout digest to 692973e [Self-hosted Renovate Bot]
+
 # v4.1.2
 ## (2024-06-24)
 

VERSION

@@ -1 +1 @@
-4.1.2
+4.1.25

balena.yml

@@ -23,4 +23,4 @@ data:
     - generic-amd64
     - genericx86-64-ext
     - intel-nuc
-version: 4.1.2
+version: 4.1.25

docker-compose.yml

@@ -79,7 +79,7 @@ services:
       *with-default-privileges,
       *with-default-volumes,
     ]
-    image: balena/open-balena-api:v25.1.7
+    image: balena/open-balena-api:v25.1.29
     depends_on:
       - db
       - redis
@@ -119,7 +119,7 @@ services:
       *with-default-healthcheck,
       *with-default-privileges,
     ]
-    image: balena/open-balena-registry:v2.39.58
+    image: balena/open-balena-registry:v2.39.60
     volumes:
       - certs-data:/certs
       - resin-data:/balena
@@ -146,7 +146,7 @@ services:
       *with-network-privileges,
       *with-default-privileges,
     ]
-    image: balena/open-balena-vpn:v11.30.22
+    image: balena/open-balena-vpn:v11.30.23
     depends_on:
       - api
     environment:
@@ -175,7 +175,7 @@ services:
       *with-default-healthcheck,
       *with-default-privileges,
     ]
-    image: balena/open-balena-s3:v2.28.45
+    image: balena/open-balena-s3:v2.28.48
     volumes:
       - s3-data:/export
       - certs-data:/certs

src/balena-tests/Dockerfile

@@ -1,7 +1,7 @@
 FROM ubuntu:24.04
 
 # renovate: datasource=github-releases depName=balena-io/balena-cli
-ARG BALENA_CLI_VERSION=v18.2.10
+ARG BALENA_CLI_VERSION=v18.2.20
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
 	bash \

src/balena-tests/balena.sh

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 
 # shellcheck disable=SC2154,SC2034,SC1090
-set -ae
+set -aeu
 
 curl_opts="--retry 3 --fail"
 if [[ $VERBOSE =~ on|On|Yes|yes|true|True ]]; then
@@ -17,7 +17,7 @@ function remove_test_assets() {
     rm -rf /balena/config.json \
       "${GUEST_IMAGE}" \
       "${GUEST_IMAGE%.*}.ready" \
-      "${tmpbuild}" \
+      "${tmpbuild:-}" \
       /tmp/*.img
 }
 
@@ -39,14 +39,16 @@ function shutdown_dut() {
     local balena_device_uuid
     balena_device_uuid="$(cat </balena/config.json | jq -r .uuid)"
 
-    if [[ -n $balena_device_uuid ]]; then
+    if [[ -n "${balena_device_uuid:-}" ]]; then
         with_backoff balena device "${balena_device_uuid}"
-        with_backoff balena device shutdown -f "${balena_device_uuid}"
+        if ! with_backoff balena device shutdown -f "${balena_device_uuid}"; then
+            echo 'DUT failed to shutdown properly'
+        fi
     fi
 }
 
 function set_update_lock {
-    if [[ -n "$BALENA_SUPERVISOR_ADDRESS" ]] && [[ -n "$BALENA_SUPERVISOR_API_KEY" ]]; then
+    if [[ -n "${BALENA_SUPERVISOR_ADDRESS:-}" ]] && [[ -n "${BALENA_SUPERVISOR_API_KEY:-}" ]]; then
         while [[ $(curl ${curl_opts} "${BALENA_SUPERVISOR_ADDRESS}/v1/device?apikey=${BALENA_SUPERVISOR_API_KEY}" \
           -H "Content-Type: application/json" | jq -r '.update_pending') == 'true' ]]; do
 
@@ -88,6 +90,7 @@ function update_ca_certificates() {
 
 function wait_for_api() {
     while ! curl ${curl_opts} "https://api.${DNS_TLD}/ping"; do
+        echo 'waiting for API...'
         sleep "$(( (RANDOM % 5) + 5 ))s"
     done
 }
@@ -96,6 +99,7 @@ function open_balena_login() {
     while ! balena login --credentials \
       --email "${SUPERUSER_EMAIL}" \
       --password "${SUPERUSER_PASSWORD}"; do
+        echo 'waiting for auth...'
         sleep "$(( (RANDOM % 5) + 5 ))s"
     done
 }
@@ -104,6 +108,7 @@ function create_fleet() {
     if ! balena fleet "${TEST_FLEET}"; then
         # wait for API to load DT contracts
         while ! balena fleet create "${TEST_FLEET}" --type "${DEVICE_TYPE}"; do
+            echo 'waiting for device types...'
             sleep "$(( (RANDOM % 5) + 5 ))s"
         done
 
@@ -175,7 +180,7 @@ function wait_for_device() {
 }
 
 function registry_auth() {
-    if [[ -n $REGISTRY_USER ]] && [[ -n $REGISTRY_PASS ]]; then
+    if [[ -n "${REGISTRY_USER:-}" ]] && [[ -n "${REGISTRY_PASS:-}" ]]; then
         with_backoff docker login -u "${REGISTRY_USER}" -p "${REGISTRY_PASS}"
 
         printf '{"https://index.docker.io/v1/": {"username":"%s", "password":"$s"}}' \
@@ -224,7 +229,7 @@ function supervisor_update_target_state() {
     local balena_device_uuid
     balena_device_uuid="$(cat </balena/config.json | jq -r .uuid)"
 
-    if [[ -n $balena_device_uuid ]]; then
+    if [[ -n "${balena_device_uuid:-}" ]]; then
         while ! curl ${curl_opts} "https://api.${DNS_TLD}/supervisor/v1/update" \
           --header "Content-Type: application/json" \
           --header "Authorization: Bearer $(cat <~/.balena/token)" \
@@ -241,9 +246,9 @@ function check_running_release() {
 
     local should_be_running_release
     should_be_running_release="$(get_release_commit)"
-    [[ -z $should_be_running_release ]] && false
+    [[ -z "$should_be_running_release" ]] && false
 
-    if [[ -n $balena_device_uuid ]]; then
+    if [[ -n "${balena_device_uuid:-}" ]]; then
         while ! [[ $(balena device "${balena_device_uuid}" | grep -E ^COMMIT | awk '{print $2}') =~ ${should_be_running_release} ]]; do
             running_release_id="$(balena device "${balena_device_uuid}" | grep -E ^COMMIT | awk '{print $2}')"
             printf 'please wait, device %s should be running %s, but is still running %s...\n' \
@@ -266,7 +271,7 @@ function get_os_version() {
 }
 
 function upload_release_asset() {
-    if [[ "$RELEASE_ASSETS_TEST" =~ true ]]; then
+    if [[ "${RELEASE_ASSETS_T:-}" =~ true ]]; then
         local release_id
         release_id=${1:-1}
         release_asset="$(find / -type f -name '*.png' | head -n 1)"
@@ -282,11 +287,11 @@ function upload_release_asset() {
 }
 
 # --- main
-if [[ "$PRODUCTION_MODE" =~ true ]]; then
+if [[ "${PRODUCTION_MODE:-}" =~ true ]]; then
     exit
 fi
 
-if [[ -n "${BALENA_DEVICE_UUID}" ]]; then
+if [[ -n "${BALENA_DEVICE_UUID:-}" ]]; then
     # prepend the device UUID if running on balenaOS
     TLD="${BALENA_DEVICE_UUID}.${DNS_TLD}"
 else
@@ -303,7 +308,19 @@ GUEST_IMAGE=${GUEST_IMAGE:-/balena/balena.img}
 OS_VERSION="$(get_os_version)"
 TEST_FLEET=${TEST_FLEET:-test-fleet}
 
-[[ -f "$CONF" ]] && source "${CONF}"
+# wait here until global config is ready
+until [[ -s "$CONF" ]]; do
+    echo 'waiting for config...'
+    sleep "$(( (RANDOM % 5) + 5 ))s"
+done
+source "${CONF}"
+
+# wait her until we have valid login credentials
+until [[ -n "${SUPERUSER_EMAIL:-}" ]] && [[ -n "${SUPERUSER_PASSWORD:-}" ]]; do
+    echo 'waiting for credentials...'
+    sleep "$(( (RANDOM % 5) + 5 ))s"
+    source "${CONF}"
+done
 
 update_ca_certificates  # ensure self-signed root CA certificate(s) trust