ExternalVendorCode/open-balena
1
0
Fork 0
mirror of https://github.com/balena-io/open-balena.git synced 2024-12-18 13:26:25 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

openBalena 2024

Browse Source 
* integration/e2e tests
* automatic SSL/TLS PKI (wildcard cert.) generation via DNS-01 challenge
* update getting started guide
* remove unnecessary privileges (Redis)
* pin Redis to v7.2 (BSD license)
* enable trust proxy

Co-authored-by: Kyle Harding <kyle@balena.io>

change-type: major
This commit is contained in:
ab77 2022-02-23 18:09:48 -08:00 committed by Anton Belodedenko
parent 86bd7facc9
commit 3553999912
No known key found for this signature in database
GPG Key ID: D094F44E5E29445A
51 changed files with 2425 additions and 1907 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
.github/workflows/flowzone.yml vendored
View File

@ -4,7 +4,6 @@ on:
pull_request:
types: [opened, synchronize, closed]
branches: [main, master]
# allow external contributions to use secrets within trusted code
pull_request_target:
types: [opened, synchronize, closed]
branches: [main, master]
@ -24,4 +23,35 @@ jobs:
)
secrets: inherit
with:
cloudflare_website: "open-balena"
jobs_timeout_minutes: 60
cloudflare_website: open-balena
custom_runs_on: |
[
[
"self-hosted",
"Linux",
"X64"
]
]
balena_slugs: |
balena/open-balena
tests:
name: openBalena tests
uses: ./.github/workflows/tests.yml
needs: [flowzone]
if: |
((
github.event.pull_request.head.repo.full_name == github.repository &&
github.event_name == 'pull_request'
) || (
github.event.pull_request.head.repo.full_name != github.repository &&
github.event_name == 'pull_request_target'
)) && github.event.action != 'closed'
secrets: inherit
with:
environment: balena-cloud.com
fleet: balena/open-balena
# https://dash.cloudflare.com/001b3ed2352612aaa068aca1b0022736/balena-devices.com/dns
dns_tld: balena-devices.com

676
.github/workflows/tests.yml vendored Normal file
View File

@ -0,0 +1,676 @@
---
name: openBalena tests
on:
workflow_call:
inputs:
environment:
description: "balenaCloud environment"
required: true
type: string
fleet:
description: "balenaCloud fleet"
required: true
type: string
dns_tld:
description: "domain name to use for issuing SSL certificates"
required: true
type: string
# https://docs.github.com/en/actions/security-guides/automatic-token-authentication
# https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#permissions
permissions:
actions: read
checks: read
contents: read
deployments: read
id-token: write # AWS GitHub OIDC required: write
issues: read
discussions: read
packages: read
pages: read
pull-requests: read
repository-projects: read
security-events: read
statuses: read
env:
# Stack ID
# arn:aws:cloudformation:us-east-1:491725000532:stack/balena-tests-s3-certs/814dea60-404d-11ed-b06f-0a7d458f8ba5
AWS_S3_CERTS_BUCKET: balena-tests-certs
# (kvm) nested virtualisation not supported on AWS/EC2 instance types|classes other than X.metal
AWS_EC2_INSTANCE_TYPE: c6a.2xlarge
AWS_EC2_LAUNCH_TEMPLATE: lt-02e10a4f66261319d
AWS_EC2_LT_VERSION: 2
AWS_IAM_USERNAME: balena-tests-iam-User-1GXO3XP12N6LL
AWS_VPC_SECURITY_GROUP_IDS: sg-057937f4d89d9d51c
AWS_VPC_SUBNET_IDS: 'subnet-02d18a08ea4058574 subnet-0a026eae1df907a09'
# otherwise it tries to send data to an endpoint provided by a private project
# https://github.com/balena-io/analytics-backend
# .. which is not part of openBalena
BALENARC_NO_ANALYTICS: '1' # https://github.com/balena-io/balena-cli/blob/master/lib/events.ts#L62-L70
DEBUG: '0' # https://github.com/balena-io/balena-cli/issues/2447
RETRY: 3
SUBDOMAIN: auto
jobs:
test:
runs-on: ["self-hosted", "X64", "distro:jammy"] # tests require socat v1.7.4
timeout-minutes: 60
strategy:
fail-fast: true
steps:
- uses: actions/checkout@44c2b7a8a4ea60a981eaca3cf939b5f4305c123b
with:
# FIXME: remove once balenaBlocks/balenaVirt is a thing
submodules: true
- uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502
with:
aws-region: ${{ vars.AWS_REGION || 'us-east-1' }}
role-session-name: github-${{ github.job }}-${{ github.run_id }}-${{ github.run_attempt }}
# balena-io/environments-bases: aws/balenacloud/ephemeral-tests/balena-tests-iam.yml
role-to-assume: ${{ vars.AWS_IAM_ROLE }}
# https://github.com/pdcastro/ssh-uuid#why
# https://github.com/pdcastro/ssh-uuid#linux-debian-ubuntu-others
- name: install additional dependencies
shell: bash
run: |
set -ue
echo '::notice::install additional dependencies'
[[ '${{ vars.VERBOSE || 'false' }}' =~ on|On|Yes|yes|true|True ]] && set -x
source src/balena-tests/functions
mkdir -p "${RUNNER_TEMP}/ssh-uuid"
wget -q -O "${RUNNER_TEMP}/ssh-uuid/ssh-uuid" https://raw.githubusercontent.com/pdcastro/ssh-uuid/master/ssh-uuid.sh \
&& chmod +x "${RUNNER_TEMP}/ssh-uuid/ssh-uuid" \
&& ln -s "${RUNNER_TEMP}/ssh-uuid/ssh-uuid" "${RUNNER_TEMP}/ssh-uuid/scp-uuid"
with_backoff balena login --token '${{ secrets.BALENA_API_KEY }}'
balena version
"${RUNNER_TEMP}/ssh-uuid/scp-uuid" --help
grep -q "${RUNNER_TEMP}/ssh-uuid" "${GITHUB_PATH}" \
|| echo "${RUNNER_TEMP}/ssh-uuid" >> "${GITHUB_PATH}"
- name: (pre)register test device
id: register-test-device
if: ${{ github.event_name == 'pull_request' && github.event.action != 'closed'}}
run: |
set -ue
[[ '${{ vars.VERBOSE || 'false' }}' =~ on|On|Yes|yes|true|True ]] && set -x
source src/balena-tests/functions
with_backoff balena login --token '${{ secrets.BALENA_API_KEY }}'
balena_device_uuid="$(openssl rand -hex 16)"
# https://www.balena.io/docs/learn/more/masterclasses/advanced-cli/#52-preregistering-a-device
balena device register '${{ inputs.fleet }}' --uuid "${balena_device_uuid}"
device_id="$(balena device "${balena_device_uuid}" | grep ^ID: | cut -c20-)"
# the actual version deployed depends on the AWS EC2/AMI, defined in AWS_EC2_LAUNCH_TEMPLATE
os_version="$(balena os versions ${{ vars.DEVICE_TYPE || 'generic-amd64' }} | head -n 1)"
balena config generate \
--version "${os_version}" \
--device "${balena_device_uuid}" \
--network ethernet \
--appUpdatePollInterval 10 \
$([[ '${{ vars.DEVELOPMENT_MODE || 'false' }}' =~ true ]] && echo '--dev') \
--output config.json
balena tag set balena ephemeral-test-device --device "${balena_device_uuid}"
github_vars=(GITHUB_ACTOR GITHUB_BASE_REF GITHUB_HEAD_REF GITHUB_JOB \
GITHUB_REF GITHUB_REF_NAME GITHUB_REF_TYPE GITHUB_REPOSITORY \
GITHUB_REPOSITORY_OWNER GITHUB_RUN_ATTEMPT GITHUB_RUN_ID GITHUB_RUN_NUMBER \
GITHUB_SHA GITHUB_WORKFLOW RUNNER_ARCH RUNNER_NAME RUNNER_OS)
for github_var in "${github_vars[@]}"; do
balena tag set ${github_var} "${!github_var}" --device "${balena_device_uuid}"
done
echo "balena_device_uuid=${balena_device_uuid}" >> "${GITHUB_OUTPUT}"
echo "balena_device_id=${device_id}" >> "${GITHUB_OUTPUT}"
# https://github.com/balena-io/balena-cli/issues/1543
- name: pin device to draft release
if: ${{ github.event_name == 'pull_request' && github.event.action != 'closed'}}
run: |
set -uae
[[ '${{ vars.VERBOSE || 'false' }}' =~ on|On|Yes|yes|true|True ]] && set -x
source src/balena-tests/functions
with_backoff balena login --token '${{ secrets.BALENA_API_KEY }}'
pr_id='${{ github.event.pull_request.id }}'
head_sha='${{ github.event.pull_request.head.sha || github.event.head_commit.id }}'
release_id="$(with_backoff balena releases '${{ inputs.fleet }}' --json \
| jq -r --arg pr_id "${pr_id}" --arg head_sha "${head_sha}" '.[]
| select(.release_tag[].tag_key=="balena-ci-commit-sha")
| select(.release_tag[].value==$head_sha)
| select(.release_tag[].tag_key=="balena-ci-id")
| select(.release_tag[].value==$pr_id).commit')"
with_backoff balena device pin \
${{ steps.register-test-device.outputs.balena_device_uuid }} \
"${release_id}"
with_backoff balena device ${{ steps.register-test-device.outputs.balena_device_uuid }}
- name: configure test device environment
if: ${{ github.event_name == 'pull_request' && github.event.action != 'closed'}}
run: |
set -ue
[[ '${{ vars.VERBOSE || 'false' }}' =~ on|On|Yes|yes|true|True ]] && set -x
source src/balena-tests/functions
with_backoff balena login --token '${{ secrets.BALENA_API_KEY }}'
with_backoff balena env add VERBOSE "${{ vars.VERBOSE || 'false' }}" \
--device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
with_backoff balena env add BALENARC_NO_ANALYTICS '1' \
--device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
with_backoff balena env add DNS_TLD '${{ env.SUBDOMAIN }}.${{ inputs.dns_tld }}' \
--device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
with_backoff balena env add DB_HOST db \
--service api \
--device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
with_backoff balena env add REDIS_HOST redis:6379 \
--service api \
--device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
# to allow devices running locally to communicate to the local API, we can route
# to the local Docker network aliases instead of public DNS, since (a) DNS_TLD is
# guestfwd(ed) in QEMU to a special internal IP 10.0.2.100; (b) is proxied to
# haproxy network alias on device; and (c) made public with a wildcard DNS record
# (e.g.)
#
# $ dig +short api.auto.balena-devices.com
# 10.0.2.100
#
with_backoff balena env add API_HOST 'api.${{ env.SUBDOMAIN }}.${{ inputs.dns_tld }}' \
--device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
# not used but required for config.json to be valid
with_backoff balena env add DELTA_HOST 'delta.${{ env.SUBDOMAIN }}.${{ inputs.dns_tld }}' \
--device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
with_backoff balena env add REGISTRY2_HOST 'registry2.${{ env.SUBDOMAIN }}.${{ inputs.dns_tld }}' \
--device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
with_backoff balena env add VPN_HOST 'cloudlink.${{ env.SUBDOMAIN }}.${{ inputs.dns_tld }}' \
--device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
with_backoff balena env add HOST 'api.${{ env.SUBDOMAIN }}.${{ inputs.dns_tld }}' \
--service api \
--device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
with_backoff balena env add TOKEN_AUTH_CERT_ISSUER 'api.${{ env.SUBDOMAIN }}.${{ inputs.dns_tld }}' \
--service api \
--device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
with_backoff balena env add REGISTRY2_TOKEN_AUTH_ISSUER 'api.${{ env.SUBDOMAIN }}.${{ inputs.dns_tld }}' \
--service registry \
--device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
with_backoff balena env add REGISTRY2_TOKEN_AUTH_REALM 'https://api.${{ env.SUBDOMAIN }}.${{ inputs.dns_tld }}/auth/v1/token' \
--service registry \
--device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
with_backoff balena env add REGISTRY2_S3_REGION_ENDPOINT 's3.${{ env.SUBDOMAIN }}.${{ inputs.dns_tld }}' \
--device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
with_backoff balena env add WEBRESOURCES_S3_HOST 's3.${{ env.SUBDOMAIN }}.${{ inputs.dns_tld }}' \
--device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
# https://github.com/balena-io/cert-manager/blob/master/entry.sh#L255-L278
# cert-manager will restore the last wildcard certificate from AWS/S3 to avoid
# being rate limited by LetsEncrypt/ACME
with_backoff balena env add AWS_S3_BUCKET '${{ env.AWS_S3_CERTS_BUCKET }}' \
--service cert-manager \
--device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
# FIXME: still required?
with_backoff balena env add COMMON_REGION '${{ env.AWS_REGION }}' \
--device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
with_backoff balena env add SUPERUSER_EMAIL 'admin@${{ env.SUBDOMAIN }}.${{ inputs.dns_tld }}' \
--device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
with_backoff balena env add ORG_UNIT openBalena \
--device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
# unstable/unsupported functionality
with_backoff balena env add HIDE_UNVERSIONED_ENDPOINT 'false' \
--service api \
--device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
with_backoff balena env add RELEASE_ASSETS_TEST 'true' \
--service sut \
--device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
- name: configure test device secrets
if: ${{ github.event_name == 'pull_request' && github.event.action != 'closed'}}
run: |
set -ue
[[ '${{ vars.VERBOSE || 'false' }}' =~ on|On|Yes|yes|true|True ]] && set -x
source src/balena-tests/functions
with_backoff balena login --token '${{ secrets.BALENA_API_KEY }}'
# cert-manager requires it to get whoami information for the user
with_backoff balena env add API_TOKEN '${{ secrets.BALENA_API_KEY }}' \
--service cert-manager \
--device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
# cert-manager requires is to request wildcard SSL certificate from LetsEncrypt
with_backoff balena env add CLOUDFLARE_API_TOKEN '${{ secrets.CLOUDFLARE_API_TOKEN }}' \
--service cert-manager \
--device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
# AWS credentials to backup/restore PKI assets
with_backoff balena env add AWS_ACCESS_KEY_ID '${{ env.AWS_ACCESS_KEY_ID }}' \
--service cert-manager \
--device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
with_backoff balena env add AWS_SECRET_ACCESS_KEY '${{ env.AWS_SECRET_ACCESS_KEY }}' \
--service cert-manager \
--device '${{ steps.register-test-device.outputs.balena_device_uuid }}'
- name: provision ephemeral test device
id: provision-test-device
if: ${{ github.event_name == 'pull_request' && github.event.action != 'closed'}}
run: |
set -ue
[[ '${{ vars.VERBOSE || 'false' }}' =~ on|On|Yes|yes|true|True ]] && set -x
for subnet_id in ${{ env.AWS_VPC_SUBNET_IDS }}; do
# spot, on-demand
for market_type in ${{ vars.MARKET_TYPES || 'spot' }}; do
# https://docs.aws.amazon.com/cli/latest/reference/ec2/run-instances.html
response="$(aws ec2 run-instances \
--launch-template 'LaunchTemplateId=${{ env.AWS_EC2_LAUNCH_TEMPLATE }},Version=${{ env.AWS_EC2_LT_VERSION }}' \
--instance-type '${{ env.AWS_EC2_INSTANCE_TYPE }}' \
$([[ $market_type =~ spot ]] && echo '--instance-market-options MarketType=spot') \
--security-group-ids '${{ env.AWS_VPC_SECURITY_GROUP_IDS }}' \
--subnet-id "${subnet_id}" \
--associate-public-ip-address \
--user-data file://config.json \
--tag-specifications "ResourceType=instance,Tags=[{Key=Name,Value=balena-tests},{Key=MarketType,Value=${market_type}},{Key=Owner,Value=${{ env.AWS_IAM_USERNAME }}},{Key=GITHUB_SHA,Value=${GITHUB_SHA}-tests}]" || true)"
[[ -n $response ]] && break
done
[[ -n $response ]] && break
done
[[ -z $response ]] && exit 1
instance_id="$(echo "${response}" | jq -r '.Instances[].InstanceId')"
aws ec2 wait instance-running --instance-ids "${instance_id}"
aws ec2 wait instance-status-ok --instance-ids "${instance_id}"
echo "instance_id=${instance_id}" >> "${GITHUB_OUTPUT}"
env:
AWS_ACCESS_KEY_ID: ${{ env.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ env.AWS_SECRET_ACCESS_KEY }}
AWS_DEFAULT_REGION: ${{ env.AWS_REGION }}
- name: provision SSH key
id: provision-ssh-key
# wait for cloud-config
# https://github.com/balena-os/cloud-config
timeout-minutes: 5
if: ${{ github.event_name == 'pull_request' && github.event.action != 'closed'}}
run: |
set -ue
[[ '${{ vars.VERBOSE || 'false' }}' =~ on|On|Yes|yes|true|True ]] && set -x
source src/balena-tests/functions
with_backoff balena login --token '${{ secrets.BALENA_API_KEY }}'
if ! [[ -e "${HOME}/.ssh/id_rsa" ]]; then
ssh-keygen -N '' \
-C "$(balena whoami | grep EMAIL | cut -c11-)" \
-f "${HOME}/.ssh/id_rsa"
fi
echo "::notice::check $(balena keys | wc -l) keys"
match=''
for key in $(balena keys | grep -v ID | awk '{print $1}'); do
fp=$(balena key ${key} | tail -n 1 | ssh-keygen -E md5 -lf /dev/stdin | awk '{print $2}')
if [[ $fp =~ $(ssh-keygen -E md5 -lf "${HOME}/.ssh/id_rsa" | awk '{print $2}') ]]; then
match="${key}"
break
fi
done
if [[ -z $match ]]; then
balena key add "${GITHUB_SHA}" "${HOME}/.ssh/id_rsa.pub"
else
balena keys
fi
pgrep ssh-agent || ssh-agent -a "${SSH_AUTH_SOCK}"
ssh-add "${HOME}/.ssh/id_rsa"
while ! [[ "$(ssh-uuid -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
${{ steps.register-test-device.outputs.balena_device_uuid }}.balena \
cat /mnt/boot/config.json | jq -r .uuid)" =~ ${{ steps.register-test-device.outputs.balena_device_uuid }} ]]; do
echo "::warning::Still working..."
sleep "$(( (RANDOM % 5) + 5 ))s"
done
echo "key_id=${GITHUB_SHA}" >> "${GITHUB_OUTPUT}"
env:
SSH_AUTH_SOCK: /tmp/ssh_agent.sock
- name: wait for application
timeout-minutes: 10
if: ${{ github.event_name == 'pull_request' && github.event.action != 'closed'}}
run: |
set -ue
[[ '${{ vars.VERBOSE || 'false' }}' =~ on|On|Yes|yes|true|True ]] && set -x
source src/balena-tests/functions
with_backoff balena login --token '${{ secrets.BALENA_API_KEY }}'
balena whoami && ssh-add -l
while [[ "$(curl -X POST --silent --retry ${{ env.RETRY }} --fail \
'https://api.${{ inputs.environment }}/supervisor/v1/device' \
--header 'authorization: Bearer ${{ secrets.BALENA_API_KEY }}' \
--header 'Content-Type:application/json' \
--data '{"uuid": "${{ steps.register-test-device.outputs.balena_device_uuid }}", "method": "GET"}' \
--compressed | jq -r '.update_pending')" =~ ^true$ ]]; do
sleep "$(( ( RANDOM % ${{ env.RETRY }} ) + ${{ env.RETRY }} ))s"
done
# wait for services to start running
while with_backoff ssh-uuid -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
${{ steps.register-test-device.outputs.balena_device_uuid }}.balena \
'balena ps -q | xargs balena inspect | jq -r .[].State.Status' \
| grep -E 'created|restarting|removing|paused|exited|dead'; do
echo "::warning::Still working..."
sleep "$(( (RANDOM % 30) + 30 ))s"
done
# wait for Docker healthchecks
while with_backoff ssh-uuid -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
${{ steps.register-test-device.outputs.balena_device_uuid }}.balena \
'balena ps -q | xargs balena inspect \
| jq -r ".[] | select(.State.Health.Status!=null).Name + \":\" + .State.Health.Status"' \
| grep -E ':starting|:unhealthy'; do
echo "::warning::Still working..."
sleep "$(( (RANDOM % 30) + 30 ))s"
done
env:
SSH_AUTH_SOCK: /tmp/ssh_agent.sock
# (TBC) https://www.balena.io/docs/reference/supervisor/docker-compose/
# due to lack of long form depends_on support in compositions, restart to ensure all
# components are running with the latest configuration; preferred over restart via
# Supervisor API restart due to potential HTTP [timeouts](https://github.com/balena-os/balena-supervisor/issues/1157)
- name: restart components
timeout-minutes: 10
if: ${{ github.event_name == 'pull_request' && github.event.action != 'closed'}}
run: |
set -ue
[[ '${{ vars.VERBOSE || 'false' }}' =~ on|On|Yes|yes|true|True ]] && set -x
source src/balena-tests/functions
with_backoff balena login --token '${{ secrets.BALENA_API_KEY }}'
balena whoami && ssh-add -l
with_backoff ssh-uuid -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
${{ steps.register-test-device.outputs.balena_device_uuid }}.balena \
"balena ps -aq | xargs balena inspect \
| jq -re '.[]
| select(.Name | contains(\"_supervisor\") | not).Id' \
| xargs balena restart"
# wait for Docker healthchecks
while with_backoff ssh-uuid -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
${{ steps.register-test-device.outputs.balena_device_uuid }}.balena \
'balena ps -q | xargs balena inspect \
| jq -r ".[] | select(.State.Health.Status!=null).Name + \":\" + .State.Health.Status"' \
| grep -E ':starting|:unhealthy'; do
echo "::warning::Still working..."
sleep "$(( (RANDOM % 30) + 30 ))s"
done
env:
SSH_AUTH_SOCK: /tmp/ssh_agent.sock
- name: SUT&DUT
if: ${{ github.event_name == 'pull_request' && github.event.action != 'closed'}}
timeout-minutes: 20
# https://giters.com/gfx/example-github-actions-with-tty
# https://github.com/actions/runner/issues/241#issuecomment-924327172
shell: 'script -q -e -c "bash {0}"'
run: |
set -ue
[[ '${{ vars.VERBOSE || 'false' }}' =~ on|On|Yes|yes|true|True ]] && set -x
source src/balena-tests/functions
with_backoff balena login --token '${{ secrets.BALENA_API_KEY }}'
balena whoami && ssh-add -l
(with_backoff ssh-uuid -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
${{ steps.register-test-device.outputs.balena_device_uuid }}.balena \
"balena ps -aq | xargs balena inspect \
| jq -re '.[] | select(.Name | contains(\"sut_\")).Id' \
| xargs balena logs -f") &
# tests service is working while its status == running
status=''
while [[ "$status" =~ Running ]]; do
status="$(curl --silent --retry ${{ env.RETRY }} --fail \
'https://api.${{ inputs.environment }}/supervisor/v2/applications/state' \
--header 'authorization: Bearer ${{ secrets.BALENA_API_KEY }}' \
--header 'Content-Type:application/json' \
--data '{"uuid": "${{ steps.register-test-device.outputs.balena_device_uuid }}", "method": "GET"}' \
--compressed | jq -r '.[].services.sut.status')"
echo "::warning::Still working..."
sleep "$(( ( RANDOM % ${{ env.RETRY }} ) + ${{ env.RETRY }} ))s"
done
# .. once the service exits with status == exited, it is assumed to be finished
status=''
while ! [[ "$status" =~ exited ]]; do
echo "::warning::Still working..."
status="$(curl --silent --retry ${{ env.RETRY }} --fail \
'https://api.${{ inputs.environment }}/supervisor/v2/applications/state' \
--header 'authorization: Bearer ${{ secrets.BALENA_API_KEY }}' \
--header 'Content-Type:application/json' \
--data '{"uuid": "${{ steps.register-test-device.outputs.balena_device_uuid }}", "method": "GET"}' \
--compressed | jq -r '.[].services.sut.status')"
sleep "$(( ( RANDOM % ${{ env.RETRY }} ) + ${{ env.RETRY }} ))s"
done
# .. check its exit code
expected_exit_code=0
actual_exit_code="$(with_backoff ssh-uuid -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
${{ steps.register-test-device.outputs.balena_device_uuid }}.balena \
"balena ps -aq | xargs balena inspect \
| jq -re '.[] | select(.Name | contains(\"sut_\")).State.ExitCode'")"
[[ $expected_exit_code -eq $actual_exit_code ]] || false
env:
SSH_AUTH_SOCK: /tmp/ssh_agent.sock
ATTEMPTS: 2
- name: remove SSH key
if: always()
continue-on-error: true
run: |
set -ue
[[ '${{ vars.VERBOSE || 'false' }}' =~ on|On|Yes|yes|true|True ]] && set -x
source src/balena-tests/functions
with_backoff balena login --token '${{ secrets.BALENA_API_KEY }}'
with_backoff balena keys | grep ${{ steps.provision-ssh-key.outputs.key_id }} \
| awk '{print $1}' | xargs balena key rm --yes
pgrep ssh-agent && (pgrep ssh-agent | xargs kill)
rm -f /tmp/ssh_agent.sock
- name: destroy balena test device
if: always()
continue-on-error: true
run: |
set -ue
[[ '${{ vars.VERBOSE || 'false' }}' =~ on|On|Yes|yes|true|True ]] && set -x
source src/balena-tests/functions
with_backoff balena login --token '${{ secrets.BALENA_API_KEY }}'
with_backoff balena device rm ${{ steps.register-test-device.outputs.balena_device_uuid }} --yes
env:
AWS_ACCESS_KEY_ID: ${{ env.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ env.AWS_SECRET_ACCESS_KEY }}
AWS_DEFAULT_REGION: ${{ env.AWS_REGION }}
# always destroy test EC2 instances even if the workflow is cancelled
- name: destroy AWS test device
if: always()
run: |
set -ue
[[ '${{ vars.VERBOSE || 'false' }}' =~ on|On|Yes|yes|true|True ]] && set -x
source src/balena-tests/functions
if [[ -n '${{ steps.provision-test-device.outputs.instance_id }}' ]]; then
with_backoff aws ec2 terminate-instances \
--instance-ids ${{ steps.provision-test-device.outputs.instance_id }}
fi
with_backoff aws ec2 describe-instances --filters Name=tag:GITHUB_SHA,Values=${GITHUB_SHA}-tests \
| jq -r .Reservations[].Instances[].InstanceId \
| xargs aws ec2 terminate-instances --instance-ids
stale_instances=$(mktemp)
aws ec2 describe-instances --filters \
Name=tag:Name,Values=balena-tests \
Name=instance-state-name,Values=running \
| jq -re '.Reservations[].Instances[].InstanceId + " " + .Reservations[].Instances[].LaunchTime' > ${stale_instances} || true
if test -s "${stale_instances}"; then
while IFS= read -r line; do
instance_id=$(echo ${line} | awk '{print $1}')
launch_time=$(echo ${line} | awk '{print $2}')
now=$(date +%s)
then=$(date --date ${launch_time} +%s)
days_since_launch=$(( (now - then) / 86400 ))
if [[ -n $days_since_launch ]] && [[ $days_since_launch -ge 1 ]]; then
with_backoff aws ec2 terminate-instances --instance-ids ${instance_id}
fi
done <${stale_instances}
rm -f ${stale_instances}
fi
env:
AWS_ACCESS_KEY_ID: ${{ env.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ env.AWS_SECRET_ACCESS_KEY }}
AWS_DEFAULT_REGION: ${{ env.AWS_REGION }}
# remove orphaned ACME DNS-01 validation records
# https://letsencrypt.org/docs/challenge-types/#dns-01-challenge
# FIXME: clean up older _acme-challenge.auto TXT records
- name: cleanup-dns-records
if: always()
continue-on-error: true
run: |
set -ue
[[ '${{ vars.VERBOSE || 'false' }}' =~ on|On|Yes|yes|true|True ]] && set -x
if [[ -n '${{ steps.register-test-device.outputs.balena_device_uuid }}' ]]; then
match="${{ steps.register-test-device.outputs.balena_device_uuid }}.${{ env.SUBDOMAIN }}"
zone_id="$(curl --silent --retry ${{ env.RETRY }} \
"https://api.cloudflare.com/client/v4/zones?name=${{ inputs.dns_tld }}" \
-H 'Authorization: Bearer ${{ secrets.CLOUDFLARE_API_TOKEN }}' | jq -r '.result[].id')"
for record in "$(curl --silent --retry ${{ env.RETRY }} \
"https://api.cloudflare.com/client/v4/zones/${zone_id}/dns_records" \
-H 'Authorization: Bearer ${{ secrets.CLOUDFLARE_API_TOKEN }}' \
| jq -r --arg match "${match}" '.result[] | select(((.type=="TXT") and (.name | contains($match))))' \
| base64)"; do
json="$(echo "${record}" | base64 -d | jq -r)"
id="$(echo "${json}" | jq -r .id)"
name="$(echo "${json}" | jq -r .name)"
if [[ -n $id ]] && [[ -n $name ]]; then
echo "::warning::Orphaned DNS record ${name} (${id})..."
if [[ -z $DRY_RUN ]]; then
curl -X DELETE --silent --retry ${{ env.RETRY }} \
"https://api.cloudflare.com/client/v4/zones/${zone_id}/dns_records/${id}" \
-H 'Authorization: Bearer ${{ secrets.CLOUDFLARE_API_TOKEN }}'
fi
fi
done
fi
env:
DRY_RUN: true

8
.gitignore vendored
View File

@ -1,7 +1,3 @@
.DS_Store
.project
.vagrant/
/config
/docker-compose.yml
/package-lock.json
.balena
**/.env

50
.openbalenarc
View File

@ -1,50 +0,0 @@
#!/bin/bash
alias dc="/home/vagrant/openbalena/scripts/compose"
function enter () {
if [[ $# -lt 1 ]]; then
echo "Usage: enter <service name> [command]"
echo " "
echo " Runs a [command] in the service specified."
echo " "
echo " command:"
echo " (default) /bin/bash"
echo " "
echo " example:"
echo " enter api # this will run the command '/bin/bash' in the API service, providing a shell prompt"
echo " enter api uptime # this will run the command 'uptime' in the API service, and return"
return 1
fi
service="$1"
shift
COMMAND=/bin/bash
if [[ $# -gt 0 ]]; then
COMMAND="$@"
fi
dc exec ${service} /bin/bash -c "${COMMAND}"
}
function logs () {
if [[ $# -lt 1 ]]; then
echo "Usage: logs <service name> [options]"
echo " "
echo " Shows the logs from journalctl in the service specified."
echo " "
echo " options:"
echo " -f tail the log stream"
echo " -n number of lines to take"
echo " "
echo " example:"
echo " logs api -fn100 # this will tail the API log, starting with the last 100 lines"
return 1
fi
service="$1"
shift
enter ${service} journalctl "$@"
}
cd /home/vagrant/openbalena

137
Makefile
View File

@ -1,4 +1,135 @@
.PHONY: lint
SHELL := bash
lint:
shellcheck scripts/*
# export all variables to child processes by default
export
# Include the .env file
include .env
DNS_TLD ?= $(error DNS_TLD not set)
TMPKI := $(shell mktemp)
STAGING_PKI ?= /usr/local/share/ca-certificates
PRODUCTION_MODE ?= true
ORG_UNIT ?= openBalena
SUPERUSER_EMAIL ?= admin@$(DNS_TLD)
.NOTPARALLEL: $(DOCKERCOMPOSE)
.PHONY: help
help: ## Print help message
@echo -e "$$(grep -hE '^\S+:.*##' $(MAKEFILE_LIST) | sed -e 's/:.*##\s*/:/' -e 's/^\(.\+\):\(.*\)/\\x1b[36m\1\\x1b[m:\2/' | column -c2 -t -s :)"
.PHONY: lint
lint: ## Lint shell scripts with shellcheck
find . -type f -name *.sh | xargs shellcheck
.PHONY: verify
verify: ## Ping the public API endpoint
curl --fail --retry 3 https://api.$(DNS_TLD)/ping
@printf '\n'
# Write all supported variables to .env, whether they have been provided or not.
# If they already exist in the .env they will be retained.
# The existing .env takes priority over envs provided from the command line.
.PHONY: config
config: ## Rewrite the .env config from current context (env vars + env args + existing .env)
ifneq ($(CLOUDFLARE_API_TOKEN),)
ifneq ($(GANDI_API_TOKEN),)
$(error "CLOUDFLARE_API_TOKEN and GANDI_API_TOKEN cannot both be set")
endif
endif
@rm -f .env
@echo "DNS_TLD=$(DNS_TLD)" >> .env
@echo "ORG_UNIT=$(ORG_UNIT)" >> .env
@echo "SUPERUSER_EMAIL=$(SUPERUSER_EMAIL)" >> .env
@echo "PRODUCTION_MODE=$(PRODUCTION_MODE)" >> .env
@echo "GANDI_API_TOKEN=$(GANDI_API_TOKEN)" >> .env
@echo "CLOUDFLARE_API_TOKEN=$(CLOUDFLARE_API_TOKEN)" >> .env
@echo "ACME_EMAIL=$(ACME_EMAIL)" >> .env
@echo "HAPROXY_CRT=$(HAPROXY_CRT)" >> .env
@echo "HAPROXY_KEY=$(HAPROXY_KEY)" >> .env
@echo "ROOT_CA=$(ROOT_CA)" >> .env
@$(MAKE) showenv
.PHONY: up
up: config ## Start all services
@docker compose up --build -d
@until [[ $$(docker compose ps api --format json | jq -r '.Health') =~ healthy ]]; do printf '.'; sleep 3; done
@printf '\n'
@$(MAKE) showenv
@$(MAKE) showpass
.PHONY: showenv
showenv: ## Print the current contents of the .env config
@cat <.env
@printf '\n'
.PHONY: printenv
printenv: ## Print the current environment variables
@printenv
.PHONY: showpass
showpass: ## Print the superuser password
@docker compose exec api cat config/env | grep SUPERUSER_PASSWORD
@printf '\n'
.PHONY: down
down: ## Stop all services
@docker compose stop
.PHONY: stop
stop: down ## Alias for 'make down'
.PHONY: restart
restart: ## Restart all services
@docker compose restart
.PHONY: update
update: # Pull and deploy latest changes from git
@git pull
@$(MAKE) up
.PHONY: destroy ## Stop and remove any existing containers and volumes
destroy:
@docker compose down --volumes --remove-orphans
.PHONY: clean
clean: destroy ## Alias for 'make destroy'
.PHONY: self-signed
self-signed: ## Install self-signed CA certificates
@sudo mkdir -p .balena $(STAGING_PKI)
@true | openssl s_client -showcerts -connect api.$(DNS_TLD):443 \
| awk '/BEGIN CERTIFICATE/,/END CERTIFICATE/ {print $0}' > $(TMPKI).ca
@cat <$(TMPKI).ca | openssl x509 -text \
| awk '/BEGIN CERTIFICATE/,/END CERTIFICATE/ {print $0}' > $(TMPKI).srv
@diff --suppress-common-lines --unchanged-line-format= \
$(TMPKI).srv \
$(TMPKI).ca | sudo tee $(STAGING_PKI)/ca-$(DNS_TLD).crt || true
@sudo update-ca-certificates
@cat <$(STAGING_PKI)/ca-$(DNS_TLD).crt | sudo tee .balena/ca-$(DNS_TLD).pem
# FIXME: refactor this function to use 'make up'
.PHONY: auto-pki
auto-pki: config # Start all services using LetsEncrypt and ACME
@docker compose exec cert-manager rm -f /certs/export/chain.pem
@docker compose up -d
@until docker compose logs cert-manager | grep -Eq "/certs/export/chain.pem Certificate will not expire in [0-9] days"; do printf '.'; sleep 3; done
@until docker compose logs cert-manager | grep -q "subject=CN = ${DNS_TLD}"; do printf '.'; sleep 3; done
@until docker compose logs cert-manager | grep -q "issuer=C = US, O = Let's Encrypt, CN = R3"; do printf '.'; sleep 3; done
@until [[ $$(docker compose ps haproxy --format json | jq -r '.Health') =~ healthy ]]; do printf '.'; sleep 3; done
@printf '\n'
@$(MAKE) showenv
@$(MAKE) showpass
.PHONY: pki-custom
pki-custom: up ## Alias for 'make up'
.PHONY: deploy
deploy: up ## Alias for 'make up'
.DEFAULT_GOAL = help

81
README.md
View File

@ -1,5 +1,9 @@
[![Flowzone](https://github.com/balena-io/open-balena/actions/workflows/flowzone.yml/badge.svg)](https://github.com/balena-io/open-balena/actions/workflows/flowzone.yml)
![](./docs/images/openbalena-logo.svg)
[![deploy button](https://balena.io/deploy.svg)](https://dashboard.balena-cloud.com/deploy?repoUrl=https://github.com/balena-io/open-balena)
OpenBalena is a platform to deploy and manage connected devices. Devices run
[balenaOS][balena-os-website], a host operating system designed for running
containers on IoT devices, and are managed via the [balena CLI][balena-cli],
@ -34,13 +38,17 @@ application to your device(s).
The current release of openBalena has the following minimum version requirements:
- balenaOS v2.58.3
- balena CLI v12.38.5
- balenaOS v5.2.8
- balena CLI v18.2.2
If you are updating from previous openBalena versions, ensure you update the balena
CLI and reprovision any devices to at least the minimum required versions in order
CLI and re-provision any devices to at least the minimum required versions in order
for them to be fully compatible with this release, as some features may not work.
While in-place openBalena upgrades may succeed, when performing major updates, it is
recommended for a new instance to be deployed in parallel with the existing one, followed
by copying state across and pointing a test device to the new instance.
## Documentation
@ -101,7 +109,12 @@ improvements and new functionality is planned:
## Differences between openBalena and balenaCloud
Whilst openBalena and balenaCloud share the same core technology, there are some key differences. First, openBalena is self-hosted, whereas balenaCloud is hosted by balena and therefore handles security, maintenance, scaling, and reliability of all the backend services. OpenBalena is also single user, whereas balenaCloud supports multiple users and organizations. OpenBalena also lacks some of the commercial features that define balenaCloud, such as the web-based dashboard and updates with binary container deltas.
Whilst openBalena and balenaCloud share the same core technology, there are some key
differences. First, openBalena is self-hosted, whereas balenaCloud is hosted by balena and
therefore handles security, maintenance, scaling, and reliability of all the backend
services. OpenBalena is also single user, whereas balenaCloud supports multiple users and
organizations. OpenBalena also lacks some of the commercial features that define
balenaCloud, such as the web-based dashboard and updates with binary container deltas.
The following table contains the main differences between both:
@ -117,12 +130,45 @@ The following table contains the main differences between both:
| Download images from [balena.io][balena-os-website] and configure locally via `balena-cli` | Download configured images directly from the dashboard |
| No remote device diagnostics | Remote device diagnostics |
Additionally, refer back to the [roadmap](#roadmap) above for planned but not yet implemented features.
Additionally, refer back to the [roadmap](#roadmap) above for planned but not yet
implemented features.
## License
OpenBalena is licensed under the terms of AGPL v3. See [LICENSE](https://github.com/balena-io/open-balena/blob/master/LICENSE) for details.
OpenBalena is licensed under the terms of AGPL v3. See [LICENSE] for details.
## FAQ
### How do you ensure continuity of openBalena? Are there security patches on openBalena?
openBalena is an open source initiative which is mostly driven by us, but it also gets
contributions from the community. We work to keep openBalena as up to date as our
bandwidth allows, especially with security patches. That said, we do not have a policy or
guarantee of a software release schedule. However, it is in our best interest to keep
openBalena updated and patched since we also use it for balenaCloud.
### How do you ensure the "Join" command actually works between openBalena and
balenaCloud?
The `balena join ..` command is frequently used for moving devices between openBalena,
and balenaCloud environments. This command extends `balena os configure ..`, which is the
basic tool balena uses for configuring devices.
### Is it "production ready"?
While we actually have some rather large fleets using openBalena, we consider it to be
perpetually in "beta". This means potentially introducing breaking changes between
releases.
### Can new device type be added to openBalena?
openBalena imports the following public [device-types] "out of the box". You can specify
your own contracts repository by overriding `CONTRACTS_PUBLIC_REPO_NAME`,
`CONTRACTS_PUBLIC_REPO_OWNER` and `IMAGE_STORAGE_BUCKET` environment variables on the API
service/container.
### Are there open-source UI dashboards from the community for openBalena?
Yes! Here are a few:
- [open-balena-admin / open-balena-ui](https://github.com/dcaputo-harmoni/open-balena-admin) by [dcaputo-harmoni](https://github.com/dcaputo-harmoni) who first posted about [here](https://forums.balena.io/t/open-balena-admin-an-admin-interface-for-openbalena/355324) in our Forums :)
- [open-balena-dashboard](https://github.com/Razikus/open-balena-dashboard) by [Razikus](https://github.com/Razikus)
[balena-cli]: https://github.com/balena-io/balena-cli
@ -135,29 +181,14 @@ OpenBalena is licensed under the terms of AGPL v3. See [LICENSE](https://github.
[forums]: https://forums.balena.io/c/open-balena
[getting-started]: https://balena.io/open/docs/getting-started
[issue-tracker]: https://github.com/balena-io/open-balena/issues
[LICENSE]: https://github.com/balena-io/open-balena/blob/master/LICENSE
[open-balena-admin / open-balena-ui]: https://github.com/dcaputo-harmoni/open-balena-admin
[open-balena-api]: https://github.com/balena-io/open-balena-api
[open-balena-dashboard]: https://github.com/Razikus/open-balena-dashboard
[open-balena-db]: https://github.com/balena-io/open-balena-db
[open-balena-registry]: https://github.com/balena-io/open-balena-registry
[open-balena-s3]: https://github.com/balena-io/open-balena-s3
[open-balena-vpn]: https://github.com/balena-io/open-balena-vpn
[open-balena-website]: https://balena.io/open
[pulls]: https://github.com/balena-io/open-balena/pulls
## FAQ
### How do you ensure continuity of openBalena? Are there security patches on openBalena?
openBalena is an open source initiative which is mostly driven by us, but it also gets contributions from the community. We work to keep openBalena as up to date as our bandwidth allows, especially with security patches. That said, we do not have a policy or guarantee of a software release schedule. However, it is in our best interest to keep openBalena updated and patched since we also use it for balenaCloud.
### How do you ensure the “Join” command actually works between open and cloud?
The join command is not only used for moving from openBalena to balenaCloud, but it is used daily by our developers to move devices from developments and testing instances to production, and vice versa. The join command actually wraps the os-config command, which is the basic tool balena uses for configuring devices.
### Is it “production ready”?
While we actually have some rather large fleets using openBalena, we as a company consider it still to be in Beta status. We don’t perform regular testing on the platform like we do balenaCloud, and we do not yet have feature-parity between the various services we offer.
### Can new device-types be added to openBalena?
Technically “yes”, but in a supported or balena-recommended fashion, “no”. The main reason is that until we regularly test the openBalena platform the way we do balenaCloud, there’s no scalable way for us to provide support for new device-types.
### Are there open-source UI dashboards from the community for openBalena?
Yes! Here are a few:
- [open-balena-admin / open-balena-ui](https://github.com/dcaputo-harmoni/open-balena-admin) by user [dcaputo-harmoni](https://github.com/dcaputo-harmoni) who first posted about [here](https://forums.balena.io/t/open-balena-admin-an-admin-interface-for-openbalena/355324) in our Forums :)
- [open-balena-dashboard](https://github.com/Razikus/open-balena-dashboard) by user [Razikus](https://github.com/Razikus)
[device-types]: https://github.com/balena-io/contracts/blob/master/contracts/hw.device-type

41
Vagrantfile vendored
View File

@ -1,41 +0,0 @@
Vagrant.require_version '>= 2.2.0'
Vagrant.configure('2') do |config|
config.vagrant.plugins = [
'vagrant-vbguest',
'vagrant-docker-compose'
]
config.vm.define 'openbalena'
config.vm.hostname = 'openbalena-vagrant'
config.vm.box = 'bento/ubuntu-18.04'
config.vm.network "public_network",
use_dhcp_assigned_default_route: true
config.vm.synced_folder '.', '/vagrant', disabled: true
config.vm.synced_folder '.', '/home/vagrant/openbalena'
config.ssh.forward_agent = true
config.vm.provision :docker
$provision = <<-SCRIPT
DOCKER_COMPOSE_VERSION=1.24.0
touch /home/vagrant/.bashrc
grep -Fxq 'source /home/vagrant/openbalena/.openbalenarc' /home/vagrant/.bashrc || echo 'source /home/vagrant/openbalena/.openbalenarc' >> /home/vagrant/.bashrc
curl -o- https://raw.githubusercontent.com/creationix/nvm/v0.34.0/install.sh | bash
source "/home/vagrant/.nvm/nvm.sh" # This loads nvm
nvm install 10.15.0 && nvm use 10.15.0
# Install a newer version of docker-compose
(cd /usr/local/bin; \
sudo curl -o docker-compose --silent --location https://github.com/docker/compose/releases/download/$DOCKER_COMPOSE_VERSION/docker-compose-Linux-x86_64; \
sudo chmod a+x docker-compose)
SCRIPT
config.vm.provision :shell, privileged: false, inline: $provision
end

26
balena.yml Normal file
View File

@ -0,0 +1,26 @@
name: openBalena
type: sw.application
description: https://www.balena.io/open
post-provisioning: |
[![Flowzone](https://github.com/balena-io/open-balena/actions/workflows/flowzone.yml/badge.svg)](https://github.com/balena-io/open-balena/actions/workflows/flowzone.yml)
## Getting Started
* https://open-balena.pages.dev/#getting-started
assets:
repository:
type: blob.asset
data:
url: 'https://github.com/balena-io/open-balena'
logo:
type: blob.asset
data:
url: 'https://raw.githubusercontent.com/balena-io/open-balena/master/logo.png'
data:
defaultDeviceType: generic-amd64
supportedDeviceTypes:
- generic-amd64
- genericx86-64-ext
- intel-nuc
version: 3.7.1

17
compose/common.yml
View File

@ -1,17 +0,0 @@
version: "2.0"
services:
component:
cap_add:
- SYS_ADMIN
- SYS_RESOURCE
environment:
- CONFD_BACKEND=ENV
tmpfs:
- /run
- /sys/fs/cgroup
privileged: true
system:
security_opt:
- seccomp:unconfined

31
compose/mdns.yml
View File

@ -1,31 +0,0 @@
version: "2.0"
services:
balena-mdns-publisher:
image: balena/balena-mdns-publisher:${OPENBALENA_MDNS_PUBLISHER_VERSION_TAG}
network_mode: "host"
cap_add:
- SYS_RESOURCE
- SYS_ADMIN
security_opt:
- apparmor:unconfined
tmpfs:
- /run
- /sys/fs/cgroup
# balenaOS - Required for host DBus comms. Not required for standalone Linux
labels:
io.balena.features.dbus: '1'
io.balena.features.supervisor-api: '1'
environment:
CONFD_BACKEND: ENV
# The name of the TLD to use. This *must* match certificates used for the rest of
# the resin backend (eg. that for BALENA_ROOT_CA if present).
MDNS_TLD: ${OPENBALENA_HOST_NAME}
# List of subdomains to advertise. This must include all required hosts.
MDNS_SUBDOMAINS: '["api", "db", "registry", "s3", "tunnel", "vpn"]'
# The expectation is the DBus socket to use is always at the following location.
DBUS_SESSION_BUS_ADDRESS: "unix:path=/host/run/dbus/system_bus_socket"
# Selects the interface used for incoming connections from the wider subnet.
# For NUCs, this is `eno1`. If running natively, pick the appropriate interface.
# Alternatively, keep the default commented out to autoselect.
#INTERFACE: "eno1"

187
compose/services.yml
View File

@ -1,187 +0,0 @@
version: "2.0"
volumes:
certs: {}
cert-provider: {}
db: {}
redis: {}
s3: {}
services:
api:
extends:
file: ./common.yml
service: component
image: balena/open-balena-api:${OPENBALENA_API_VERSION_TAG}
depends_on:
- db
- s3
- redis
environment:
API_VPN_SERVICE_API_KEY: ${OPENBALENA_API_VPN_SERVICE_API_KEY}
ROOT_CA: ${OPENBALENA_ROOT_CA}
COOKIE_SESSION_SECRET: ${OPENBALENA_COOKIE_SESSION_SECRET}
DB_HOST: db
DB_PASSWORD: docker
DB_PORT: 5432
DB_USER: docker
DELTA_HOST: delta.${OPENBALENA_HOST_NAME}
DEVICE_CONFIG_OPENVPN_CA: ${OPENBALENA_VPN_CA_CHAIN}
DEVICE_CONFIG_SSH_AUTHORIZED_KEYS: ${OPENBALENA_SSH_AUTHORIZED_KEYS}
HOST: api.${OPENBALENA_HOST_NAME}
IMAGE_MAKER_URL: img.${OPENBALENA_HOST_NAME}
IMAGE_STORAGE_BUCKET: resin-production-img-cloudformation
IMAGE_STORAGE_PREFIX: images
IMAGE_STORAGE_ENDPOINT: s3.amazonaws.com
JSON_WEB_TOKEN_EXPIRY_MINUTES: 10080
JSON_WEB_TOKEN_SECRET: ${OPENBALENA_JWT_SECRET}
MIXPANEL_TOKEN: __unused__
PRODUCTION_MODE: "${OPENBALENA_PRODUCTION_MODE}"
PUBNUB_PUBLISH_KEY: __unused__
PUBNUB_SUBSCRIBE_KEY: __unused__
REDIS_HOST: "redis:6379"
REDIS_IS_CLUSTER: "false"
REGISTRY2_HOST: registry.${OPENBALENA_HOST_NAME}
REGISTRY_HOST: registry.${OPENBALENA_HOST_NAME}
SENTRY_DSN: ""
TOKEN_AUTH_BUILDER_TOKEN: ${OPENBALENA_TOKEN_AUTH_BUILDER_TOKEN}
TOKEN_AUTH_CERT_ISSUER: api.${OPENBALENA_HOST_NAME}
TOKEN_AUTH_CERT_KEY: ${OPENBALENA_TOKEN_AUTH_KEY}
TOKEN_AUTH_CERT_KID: ${OPENBALENA_TOKEN_AUTH_KID}
TOKEN_AUTH_CERT_PUB: ${OPENBALENA_TOKEN_AUTH_PUB}
TOKEN_AUTH_JWT_ALGO: "ES256"
VPN_HOST: vpn.${OPENBALENA_HOST_NAME}
VPN_PORT: 443
VPN_SERVICE_API_KEY: ${OPENBALENA_VPN_SERVICE_API_KEY}
SUPERUSER_EMAIL: ${OPENBALENA_SUPERUSER_EMAIL}
SUPERUSER_PASSWORD: ${OPENBALENA_SUPERUSER_PASSWORD}
registry:
extends:
file: ./common.yml
service: component
image: balena/open-balena-registry:${OPENBALENA_REGISTRY_VERSION_TAG}
depends_on:
- s3
- redis
environment:
API_TOKENAUTH_CRT: ${OPENBALENA_TOKEN_AUTH_PUB}
REGISTRY2_HOST: registry.${OPENBALENA_HOST_NAME}
ROOT_CA: ${OPENBALENA_ROOT_CA}
REGISTRY2_TOKEN_AUTH_ISSUER: api.${OPENBALENA_HOST_NAME}
REGISTRY2_TOKEN_AUTH_REALM: https://api.${OPENBALENA_HOST_NAME}/auth/v1/token
COMMON_REGION: ${OPENBALENA_S3_REGION}
REGISTRY2_CACHE_ENABLED: "false"
REGISTRY2_CACHE_ADDR: 127.0.0.1:6379
REGISTRY2_CACHE_DB: 0
REGISTRY2_CACHE_MAXMEMORY_MB: 1024 # megabytes
REGISTRY2_CACHE_MAXMEMORY_POLICY: allkeys-lru
REGISTRY2_S3_REGION_ENDPOINT: ${OPENBALENA_S3_ENDPOINT}
REGISTRY2_S3_BUCKET: ${OPENBALENA_REGISTRY2_S3_BUCKET}
REGISTRY2_S3_KEY: ${OPENBALENA_S3_ACCESS_KEY}
REGISTRY2_S3_SECRET: ${OPENBALENA_S3_SECRET_KEY}
REGISTRY2_SECRETKEY: ${OPENBALENA_REGISTRY_SECRET_KEY}
REGISTRY2_STORAGEPATH: /data
REGISTRY2_DISABLE_REDIRECT: "false"
REGISTRY2_DISABLE_UPLOAD_PURGING: "false"
vpn:
extends:
file: ./common.yml
service: component
image: balena/open-balena-vpn:${OPENBALENA_VPN_VERSION_TAG}
depends_on:
- api
cap_add:
- NET_ADMIN
environment:
API_SERVICE_API_KEY: ${OPENBALENA_API_VPN_SERVICE_API_KEY}
API_HOST: api.${OPENBALENA_HOST_NAME}
ROOT_CA: ${OPENBALENA_ROOT_CA}
VPN_PORT: 443
PRODUCTION_MODE: "${OPENBALENA_PRODUCTION_MODE}"
VPN_GATEWAY: 10.2.0.1
SENTRY_DSN: ""
VPN_HAPROXY_USEPROXYPROTOCOL: "true"
VPN_OPENVPN_CA_CRT: ${OPENBALENA_VPN_CA}
VPN_OPENVPN_SERVER_CRT: ${OPENBALENA_VPN_SERVER_CRT}
VPN_OPENVPN_SERVER_DH: ${OPENBALENA_VPN_SERVER_DH}
VPN_OPENVPN_SERVER_KEY: ${OPENBALENA_VPN_SERVER_KEY}
VPN_SERVICE_API_KEY: ${OPENBALENA_VPN_SERVICE_API_KEY}
db:
extends:
file: ./common.yml
service: system
image: balena/open-balena-db:${OPENBALENA_DB_VERSION_TAG}
volumes:
- db:/var/lib/postgresql/data
s3:
extends:
file: ./common.yml
service: component
image: balena/open-balena-s3:${OPENBALENA_S3_VERSION_TAG}
volumes:
- s3:/export
environment:
S3_MINIO_ACCESS_KEY: ${OPENBALENA_S3_ACCESS_KEY}
S3_MINIO_SECRET_KEY: ${OPENBALENA_S3_SECRET_KEY}
BUCKETS: ${OPENBALENA_S3_BUCKETS}
redis:
extends:
file: ./common.yml
service: system
image: redis:alpine
volumes:
- redis:/data
haproxy:
extends:
file: ./common.yml
service: system
build: ../src/haproxy
depends_on:
- api
- cert-provider
- db
- s3
- redis
- registry
- vpn
ports:
- "80:80"
- "443:443"
expose:
- "222"
- "3128"
- "5432"
- "6379"
networks:
default:
aliases:
- api.${OPENBALENA_HOST_NAME}
- registry.${OPENBALENA_HOST_NAME}
- vpn.${OPENBALENA_HOST_NAME}
- db.${OPENBALENA_HOST_NAME}
- s3.${OPENBALENA_HOST_NAME}
- redis.${OPENBALENA_HOST_NAME}
- tunnel.${OPENBALENA_HOST_NAME}
environment:
BALENA_HAPROXY_CRT: ${OPENBALENA_ROOT_CRT}
BALENA_HAPROXY_KEY: ${OPENBALENA_ROOT_KEY}
BALENA_ROOT_CA: ${OPENBALENA_ROOT_CA}
HAPROXY_HOSTNAME: ${OPENBALENA_HOST_NAME}
volumes:
- certs:/certs:ro
cert-provider:
build: ../src/cert-provider
volumes:
- certs:/certs
- cert-provider:/usr/src/app/certs
environment:
ACTIVE: ${OPENBALENA_ACME_CERT_ENABLED}
DOMAINS: "api.${OPENBALENA_HOST_NAME},registry.${OPENBALENA_HOST_NAME},s3.${OPENBALENA_HOST_NAME},vpn.${OPENBALENA_HOST_NAME},tunnel.${OPENBALENA_HOST_NAME}"
OUTPUT_PEM: /certs/open-balena.pem

10
compose/template.yml
View File

@ -1,10 +0,0 @@
# Project-specific config.
#
# All paths must be defined relative to `compose/services.yml` regardless of
# the location of this file, i.e. refer to `my-open-balena-checkout/somedir`
# as `../somedir`. This is because of the way docker-compose handles paths
# when specifying multiple configs and open-balena always specifying
# `compose/services.yml` as the "base" config.
#
# You may view the effective config with `scripts/compose config`.
version: "2.0"

6
compose/versions
View File

@ -1,6 +0,0 @@
export OPENBALENA_API_VERSION_TAG=v19.1.19
export OPENBALENA_DB_VERSION_TAG=v5.2.2
export OPENBALENA_MDNS_PUBLISHER_VERSION_TAG=v1.27.99
export OPENBALENA_REGISTRY_VERSION_TAG=v2.39.45
export OPENBALENA_S3_VERSION_TAG=v2.28.32
export OPENBALENA_VPN_VERSION_TAG=v11.29.36

360
docker-compose.yml Normal file
View File

@ -0,0 +1,360 @@
---
version: '2.4'
volumes:
builder-certs-ca: {}
builder-certs-client: {}
builder-data: {}
cert-manager-data: {}
certs-data: {}
db-data: {}
pki-data: {}
redis-data: {}
resin-data: {}
s3-data: {}
x-default-healthcheck: &default-healthcheck
test: /usr/src/app/docker-hc
interval: 45s
timeout: 15s
retries: 3
x-default-environment: &default-environment
# FIXME: hardcoded https://github.com/balena-io/open-balena-db/blob/master/create-resin-db.sh#L4
DB_NAME: resin
# FIXME: hardcoded https://github.com/balena-io/open-balena-db/blob/master/Dockerfile#L3-L4
DB_PASSWORD: docker
DB_USER: docker
LOG_LEVEL: DEBUG
PRODUCTION_MODE: 'false'
x-default-healthcheck-trait: &with-default-healthcheck
healthcheck:
<<: *default-healthcheck
x-default-volumes-trait: &with-default-volumes
volumes:
- certs-data:/certs
- resin-data:/balena
x-default-privileges-trait: &with-default-privileges
cap_add:
- SYS_ADMIN
- SYS_RESOURCE
security_opt:
- apparmor=unconfined
tmpfs:
- /run
- /sys/fs/cgroup
x-extended-privileges-trait: &with-extended-privileges
security_opt:
- apparmor=unconfined
- seccomp=unconfined
x-all-privileges-trait: &with-all-privileges
privileged: true
cap_add:
- ALL
x-network-privileges-trait: &with-network-privileges
cap_add:
- NET_ADMIN
- SYS_ADMIN
- SYS_RESOURCE
x-base-service-definition: &base-service
restart: unless-stopped
# for docker-compose only, no effect on balenaCloud
env_file:
- .env
tty: 'true' # send syastemd logs from containers to stdout
services:
# https://github.com/balena-io/open-balena-api
api:
<<: [
*base-service,
*with-default-healthcheck,
*with-default-privileges,
*with-default-volumes,
]
image: balena/open-balena-api:v22.2.2
depends_on:
- db
- redis
- s3
environment:
<<: *default-environment
CONTRACTS_PUBLIC_REPO_NAME: contracts
CONTRACTS_PUBLIC_REPO_OWNER: balena-io
DB_GENERAL_REPLICA_MAX_USES: 1000
DB_GENERAL_REPLICA_PORT: 5432
DB_HOST: db
DB_PORT: 5432
DB_STATE_REPLICA_MAX_USES: 1000
DB_STATE_REPLICA_PORT: 5432
DB_USER: docker
HOSTS_CONFIG: API_HOST:api,DB_HOST:db,DELTA_HOST:delta,HOST:api,REDIS_HOST:redis,TOKEN_AUTH_CERT_ISSUER:api,VPN_HOST:cloudlink,REGISTRY2_HOST:registry2
IMAGE_STORAGE_BUCKET: resin-production-img-cloudformation
IMAGE_STORAGE_ENDPOINT: s3.amazonaws.com
IMAGE_STORAGE_PREFIX: images
JSON_WEB_TOKEN_EXPIRY_MINUTES: 10080
NUM_WORKERS: 1
OAUTH_CALLBACK_PROTOCOL: https
PORT: 80
REDIS_HOST: redis:6379
REDIS_IS_CLUSTER: 'false'
TOKEN_AUTH_JWT_ALGO: ES256
TOKENS_CONFIG: API_SERVICE_API_KEY:hex,AUTH_RESINOS_REGISTRY_CODE:hex,COOKIE_SESSION_SECRET:hex,JSON_WEB_TOKEN_SECRET:hex,MIXPANEL_TOKEN:hex,SUPERUSER_PASSWORD:hex,TOKEN_AUTH_BUILDER_TOKEN:hex,VPN_GUEST_API_KEY:hex,VPN_SERVICE_API_KEY:hex,API_VPN_SERVICE_API_KEY:API_SERVICE_API_KEY,REGISTRY2_TOKEN:TOKEN_AUTH_BUILDER_TOKEN
TRUST_PROXY: 172.16.0.0/12
VPN_PORT: 443
WEBRESOURCES_S3_BUCKET: web-resources
WEBRESOURCES_S3_REGION: "us-east-1" # this is required for minio
# https://github.com/balena-io/open-balena-registry
registry:
<<: [
*base-service,
*with-default-healthcheck,
*with-default-privileges,
]
image: balena/open-balena-registry:v2.39.55
volumes:
- certs-data:/certs
- resin-data:/balena
depends_on:
- redis
- s3
environment:
COMMON_REGION: open-balena
HOSTS_CONFIG: REGISTRY2_HOST:registry2,REGISTRY2_TOKEN_AUTH_ISSUER:api,REGISTRY2_TOKEN_AUTH_REALM:api
REGISTRY2_CACHE_ADDR: redis:6379
REGISTRY2_CACHE_DB: 1
REGISTRY2_CACHE_ENABLED: 'true'
REGISTRY2_S3_BUCKET: registry-data
REGISTRY2_STORAGEPATH: /data
TOKENS_CONFIG: REGISTRY2_SECRETKEY:hex
# https://github.com/balena-io/open-balena-vpn
vpn:
<<: [
*base-service,
*with-default-healthcheck,
*with-default-volumes,
# privileges in order from minimum to maximum
*with-network-privileges,
*with-default-privileges,
]
image: balena/open-balena-vpn:v11.30.9
depends_on:
- api
environment:
HOSTS_CONFIG: VPN_HOST:cloudlink
TOKENS_CONFIG: ','
VPN_HAPROXY_USEPROXYPROTOCOL: 'true'
VPN_PORT: 443
# ensure correct service instance IP is registered with the API
VPN_SERVICE_REGISTER_INTERFACE: eth0
# https://github.com/balena-io/open-balena-db
db:
<<: *base-service
image: balena/open-balena-db:v5.2.2
volumes:
- db-data:/var/lib/postgresql/data
environment:
<<: *default-environment
healthcheck:
test: pg_isready -U "$${DB_USER}" -d "$${DB_NAME}"
# https://github.com/balena-io/open-balena-s3
s3:
<<: [
*base-service,
*with-default-healthcheck,
*with-default-privileges,
]
image: balena/open-balena-s3:v2.28.42
volumes:
- s3-data:/export
- certs-data:/certs
- resin-data:/balena
environment:
BUCKETS: registry-data;web-resources
HOSTS_CONFIG: REGISTRY2_S3_REGION_ENDPOINT:s3,WEBRESOURCES_S3_HOST:s3
TOKENS_CONFIG: REGISTRY2_S3_KEY:hex,REGISTRY2_S3_SECRET:hex,S3_MINIO_ACCESS_KEY:REGISTRY2_S3_KEY,S3_MINIO_SECRET_KEY:REGISTRY2_S3_SECRET,WEBRESOURCES_S3_ACCESS_KEY:REGISTRY2_S3_KEY,WEBRESOURCES_S3_SECRET_KEY:REGISTRY2_S3_SECRET
# https://hub.docker.com/_/redis
redis:
<<: *base-service
# https://redis.io/blog/what-redis-license-change-means-for-our-managed-service-providers/
image: redis:7.2-alpine
volumes:
- redis-data:/data
healthcheck:
<<: *default-healthcheck
test: echo INFO | redis-cli | grep redis_version
# https://github.com/balena-io/open-balena-haproxy
haproxy:
<<: [
*base-service,
*with-default-privileges,
*with-default-volumes,
]
build: src/haproxy
sysctls:
# https://github.com/docker-library/haproxy/issues/160
net.ipv4.ip_unprivileged_port_start: 0
healthcheck:
<<: *default-healthcheck
test: true | openssl s_client -connect localhost:443
ports:
# haproxy/http
- "80:80/tcp"
# haproxy/tcp-router
- "443:443/tcp"
# haproxy/stats
- "1936:1936/tcp"
environment:
LOGLEVEL: info
# dynamically configure Docker network aliases based on DNS_TLD and ALIAS list
# allows DNS resolution from systemd-less images on the Docker network
haproxy-sidecar:
<<: *base-service
build: src/haproxy-sidecar
volumes:
- /var/run/docker.sock:/host/run/docker.sock
environment:
DOCKER_HOST: unix:///host/run/docker.sock
# resolved internally as {{service}}.{{dns-tld-without-balena-device-uuid}} to haproxy service
ALIASES: api,ca,cloudlink,db,delta,logs,redis,registry2,s3,stats,tunnel
labels:
io.balena.features.balena-socket: 1
io.balena.features.supervisor-api : 1
# https://github.com/balena-io/cert-manager
# https://certbot.eff.org/docs/using.html
# https://certbot-dns-cloudflare.readthedocs.io/
cert-manager:
<<: *base-service
build: src/cert-manager
volumes:
- cert-manager-data:/etc/letsencrypt
- certs-data:/certs
- resin-data:/balena
depends_on:
- balena-ca
environment:
# wildcard certificate for reverse proxy
SSH_KEY_NAMES: ','
SUBJECT_ALTERNATE_NAMES: '*'
labels:
io.balena.features.balena-api: 1
io.balena.features.supervisor-api: 1
# https://github.com/balena-io/ca-private
# https://github.com/cloudflare/cfssl/blob/master/doc/api/intro.txt
balena-ca:
<<: *base-service
image: balena/ca-private:v0.0.14
volumes:
- pki-data:/pki
- certs-data:/certs
- resin-data:/balena
healthcheck:
test: curl --silent -I --fail localhost:8888
interval: 60s
timeout: 60s
retries: 10
labels:
# future expansion
io.balena.features.balena-api: 1
io.balena.features.supervisor-api: 1
# --- the following are not required for runtime operation of openBalena
# only relevant when running in AWS/EC2
tag-sidecar:
build: src/tag-sidecar
restart: no
environment:
ENABLED: 'true'
labels:
io.balena.features.balena-api: 1
# Software Under Test (SUT) tests orchestrator
sut:
<<: [
*base-service,
*with-extended-privileges,
*with-network-privileges,
]
build: src/balena-tests
command: /usr/sbin/balena.sh
environment:
DOCKER_CERT_PATH: /docker-pki/client
DOCKER_HOST: docker:2376
DOCKER_TLS_VERIFY: 'true'
GUEST_IMAGE: /balena/balena.img
volumes:
- builder-certs-client:/docker-pki/client
- certs-data:/certs
- resin-data:/balena
labels:
io.balena.features.balena-api: 1
io.balena.features.supervisor-api: 1
restart: no
# virtual Device Under Test (DUT)
dut:
<<: [
*base-service,
*with-extended-privileges,
*with-network-privileges,
]
# https://hub.docker.com/r/qemux/qemu-docker
# https://github.com/qemus/qemu-docker
build: src/test-device
entrypoint:
- /bin/sh
- -c
command:
- /usr/sbin/balena.sh
environment:
GUEST_IMAGE: /balena/balena.img
MEMORY: 3072M
CPU: 4
volumes:
- resin-data:/balena
devices:
- /dev/net/tun
restart: no
# https://hub.docker.com/_/docker
# pseudo(builder) service for balena-tests
docker:
<<: [
*base-service,
*with-extended-privileges,
*with-network-privileges,
]
image: docker:dind
volumes:
- builder-data:/var/lib/docker
- builder-certs-ca:/docker-pki/ca
- builder-certs-client:/docker-pki/client
- /sys:/sys
environment:
DOCKER_TLS_CERTDIR: /docker-pki
healthcheck:
test: docker system info
interval: 60s
timeout: 60s
retries: 5
labels:
io.balena.features.sysfs: 1

529
docs/getting-started.md
View File

@ -1,158 +1,163 @@
# Openbalena Getting Started Guide
# openBalena Getting Started Guide
This guide will walk you through the steps of deploying an openBalena server,
that together with the balena CLI, will enable you to create and manage a fleet
of devices running on your own infrastructure, on premises or in the cloud. The
openBalena servers must be reachable by the devices, which is easiest to achieve
with cloud providers like AWS, Google Cloud, Digital Ocean and others.
This guide will walk you through the steps of deploying an openBalena server, that
together with the balena CLI, will enable you to create and manage a fleet of devices
running on your own infrastructure, on premises or in the cloud. The openBalena servers
must be reachable by the devices, which is easiest to achieve with cloud providers like
AWS, Google Cloud, Digital Ocean and others.
This guide assumes a setup with two separate machines:
- The openBalena _server_, running Linux. These instructions were tested with an
Ubuntu 18.04 x64 server.
- The _local machine_, running Linux, Windows or macOS where the balena CLI runs
(as a client to the openBalena server). The local machine should also have a
working installation of [Docker](https://docs.docker.com/get-docker/) so that
application images can be built and deployed to your devices, although it is
also possible to use balenaEngine on a balenaOS device instead of Docker.
- A _server_, running Linux with at least 2GB of memory. These instructions were tested
with Ubuntu 20.04, 22.04 and 24.04 x64 servers. The server must have a working
installation of [Docker Engine] and you must have root permissions.
- A _local machine_, running Linux, Windows or macOS where the balena CLI runs (as a
client to the openBalena server). The local machine must also have a working
installation of [Docker] so that application images can be built and deployed to your
device. It is also possible to use [balenaEngine] on a [balenaOS] device instead of
Docker.
### Preparing a server for openBalena
Additionally, a _device type_ and compatible flash media supported by [balenaOS]
(e.g. Raspberry Pi) are required to complete the provisioning demo. Ensure the correct
power supply is available to power this device.
Login to the server via SSH and run the following commands.
## Domain Configuration
1. First, install or update essential software:
```bash
apt-get update && apt-get install -y build-essential git docker.io libssl-dev nodejs npm
```
2. Install docker-compose:
```bash
curl -L https://github.com/docker/compose/releases/download/1.27.4/docker-compose-Linux-x86_64 -o /usr/local/bin/docker-compose
chmod +x /usr/local/bin/docker-compose
```
Test your docker-compose installation with `$ docker-compose --version`.
3. Create a new user, assign admin permissions and add to `docker` group:
```bash
adduser balena
usermod -aG sudo balena
usermod -aG docker balena
```
#### Install openBalena on the server
1. On the server still, login as the new user and change into the home directory:
```bash
su balena
cd ~
```
2. Clone the openBalena repository and change into the new directory:
```bash
git clone https://github.com/balena-io/open-balena.git
cd open-balena/
```
3. Run the `quickstart` script as below. This will create a new `config`
directory and generate appropriate SSL certificates and configuration for the
server. The provided email and password will be used to automatically create
the user account for interacting with the server and will be needed later on
for logging in via the balena CLI. Replace the domain name for the `-d`
argument appropriately.
```bash
./scripts/quickstart -U <email@address> -P <password> -d mydomain.com
```
For more available options, see the script's help:
```bash
./scripts/quickstart -h
```
4. At this point, the openBalena server can be started with:
```bash
systemctl start docker
./scripts/compose up -d
```
The `-d` argument spawns the containers as background services.
5. Tail the logs of the containers with:
```bash
./scripts/compose exec <service-name> journalctl -fn100
```
Replace `<service-name>` with the name of any one of the services defined
in `compose/services.yml`; eg. `api` or `registry`.
6. The server can be stopped with:
```bash
./scripts/compose stop
```
When updating openBalena to a new version, the steps are:
```bash
./scripts/compose down
git pull
./scripts/compose build
./scripts/compose up -d
```
#### Domain Configuration
The following CNAME records must be configured to point to the openBalena server:
The following DNS records must be configured to point to the openBalena server prior to
configuration:
```text
api.mydomain.com
registry.mydomain.com
vpn.mydomain.com
ca.mydomain.com
cloudlink.mydomain.com
logs.mydomain.com
ocsp.mydomain.com
registry2.mydomain.com
s3.mydomain.com
tunnel.mydomain.com
```
Check with your internet domain name registrar for instructions on how to
configure CNAME records.
Alternatively you may consider adding a single wildcard DNS record `*.mydomain.com`.
#### Test the openBalena server
Check with your Internet domain name registrar for instructions on how to obtain a domain
name and configure records.
To confirm that everything is running correctly, try a simple request from the
local machine to the server:
## Install openBalena on the server
1. First [Change cgroup version] to v1 for compatibility with systemd in containers on
modern Linux distributions, where cgroups v2 are enabled by default:
```bash
curl -k https://api.mydomain.com/ping
OK
source /etc/default/grub
sudo sed -i '/GRUB_CMDLINE_LINUX/d' /etc/default/grub
echo GRUB_CMDLINE_LINUX=$(printf '\"%s systemd.unified_cgroup_hierarchy=0\"\n' "${GRUB_CMDLINE_LINUX}") \
| sudo tee -a /etc/default/grub
sudo update-grub
sudo reboot
```
Congratulations! The openBalena server is up and running. The next step is to
setup the local machine to use the server, provision a device and deploy a
small project.
2. Ensure cgroups v2 is disabled
### Install self-signed certificates on the local machine
```bash
if [ ! -f /sys/fs/cgroup/cgroup.controllers ]; then
echo "cgroups v2 is disabled"
else
echo "cgroups v2 is enabled"
fi
```
The installation of the openBalena server produces a few self-signed certificates
that must be installed on the local machine, so that it can securely communicate
with the server.
3. Now, install or update essential software:
The root certificate is found at `config/certs/root/ca.crt` on the server. Copy
it to some folder on the local machine and keep a note to the path -- it will be
used later during the CLI installation. Follow the steps below for the specific
platform of the local machine.
```bash
sudo apt-get update && sudo apt-get install -y make openssl git jq
```
4. Install Docker Engine
```bash
which docker || curl -fsSL https://get.docker.com | sh -
```
5. Create a new user with appropriate permissions:
```bash
sudo useradd -s /bin/bash -m -G docker,sudo balena
echo 'balena ALL=(ALL) NOPASSWD: ALL' | tee >/etc/sudoers.d/balena
```
6. Switch user:
```bash
sudo su balena
```
7. Clone the openBalena repository and change directory:
```bash
git clone https://github.com/balena-io/open-balena.git ~/open-balena
cd ~/open-balena
```
8. Start the server on your domain name:
```bash
export DNS_TLD=mydomain.com
make up
```
Note down `SUPERUSER_EMAIL` and `SUPERUSER_PASSWORD` values to be used later.
9. Tail the logs of the containers with:
```bash
docker compose logs -f api
```
Replace `api` with the name of any one of the services from the [composition].
10. The server can be stopped with:
```bash
make down
```
The server can also be restarted using `make restart`.
To update openBalena, run:
```bash
make update
```
### Test the openBalena server
To confirm that everything is running correctly, try a simple request from the local
machine to the server after registering its CA certificate(s) with the host:
```bash
make self-signed
make verify
```
Note, if you've previously stopped the server with `make down`, run `make up` again first.
Congratulations! The openBalena server is up and running. The next step is to setup your
local machine to use this server, provision a device and deploy a small project.
### Install self-signed certificates on the local machine.
The installation of the openBalena server produces a self-signed certificate by default,
which must be trusted by all devices communicating with it. This type of configuration is
not recommended for production deployments, skip to [SSL Configuration](#ssl-configuration)
instead.
The root CA bundle can be found at `.balena/ca-${DNS_TLD}.pem` on the server. Follow the
steps below for your specific local machine platform after manually copying it across.
#### Linux:
```bash
sudo cp ca.crt /usr/local/share/ca-certificates/ca.crt
sudo cp ca.pem /usr/local/share/ca-certificates/
sudo update-ca-certificates
sudo systemctl restart docker
```
@ -160,24 +165,85 @@ sudo systemctl restart docker
#### macOS:
```bash
sudo security add-trusted-cert -d -r trustRoot -k /Library/Keychains/System.keychain ca.crt
osascript -e 'quit app "Docker"' && open -a Docker
sudo security add-trusted-cert -d \
-r trustRoot \
-k /Library/Keychains/System.keychain \
ca.pem
curl http://localhost/engine/restart \
-H 'Content-Type: application/json' \
-d '{"openContainerView": true}' \
--unix-socket ~/Library/Containers/com.docker.docker/Data/backend.sock
```
#### Windows:
```bash
certutil -addstore -f "ROOT" ca.crt
```PowerShell
certutil -addstore -f "ROOT" ca.pem
Stop-Service -Name Docker
Start-Service -Name Docker
```
The Docker daemon on the local machine must then be restarted for Docker to
pick up the new certificate.
### SSL Configuration
opeBalena server now uses automatic SSL configuration via ACME [DNS-01] challenge[^1]. Support
for the following DNS providers is currently implemented:
* Cloudflare
* Gandi
#### Cloudflare
Obtain a Cloudflare API token with write access to your openBalena domain name records:
```bash
export ACME_EMAIL=acme@mydomain.com
export CLOUDFLARE_API_TOKEN={{token}}
```
#### Gandi
Obtain a Gandi API token with write access to your openBalena domain name records:
```bash
export ACME_EMAIL=acme@mydomain.com
export GANDI_API_TOKEN={{token}}
```
#### Re-configure and test the server
```bash
make auto-pki
make verify
```
#### Custom SSL
openBalena server also supports custom/manual TLS configuration. You must supply your own
SSL certificate, private key and a full certificate signing chain. A wildcard SSL
certificate covering the whole domain is recommended.
1. After obtaining your certificate, run the following commands on openBalena server:
```bash
export HAPROXY_CRT="{{ base64 encoded server certificate }}"
export ROOT_CA="{{ .. intermediate certificates }}"
export HAPROXY_KEY="{{ .. private key }}"
```
Pipe the plaintext via `.. | openssl base64 -A` to encode.
2. Re-configure and test the server:
```bash
make pki-custom
make verify
```
### Install the balena CLI on the local machine
Follow the [balena CLI installation
instructions](https://github.com/balena-io/balena-cli/blob/master/INSTALL.md)
to install the balena CLI on the local machine.
Follow the [balena CLI installation instructions] to install the balena CLI on the local
machine.
By default, the CLI targets the balenaCloud servers at `balena-cloud.com`, and
needs to be configured to target the openBalena server instead. Add the following
@ -193,17 +259,19 @@ The CLI configuration file can be found at:
- On Linux or macOS: `~/.balenarc.yml`
- On Windows: `%UserProfile%\_balenarc.yml`
If the file does not already exist, just create it.
If the file does not already exist, just create it. Alternatively, `BALENARC_BALENA_URL`
environment variable can be set to point to `"mydomain.com"`.
Wrapping up the CLI installation, set an environment variable that points to the
root certificate copied previously on the local machine. This step is to ensure
the CLI can securely interact with the openBalena server.
the CLI can securely interact with the openBalena server when running self-signed PKI.
This step can be skipped if the server is operating with publicly trusted PKI.
| Shell | Command |
| ------------------ | ---------------------------------------------- |
| bash | `export NODE_EXTRA_CA_CERTS='/path/to/ca.crt'` |
| Windows cmd.exe | `set NODE_EXTRA_CA_CERTS=C:\path\to\ca.crt` |
| Windows PowerShell | `$Env:NODE_EXTRA_CA_CERTS="C:\path\to\ca.crt"` |
| bash | `export NODE_EXTRA_CA_CERTS='/path/to/ca.pem'` |
| Windows cmd.exe | `set NODE_EXTRA_CA_CERTS=C:\path\to\ca.pem` |
| Windows PowerShell | `$Env:NODE_EXTRA_CA_CERTS="C:\path\to\ca.pem"` |
### Deploy an application
@ -213,75 +281,99 @@ variable is set, as discussed above.
#### Login to openBalena
Run `balena login`, select `Credentials` and use the email and password
specified during quickstart to login to the openBalena server. At any time, the
`balena whoami` command may be used to check which server the CLI is logged in to.
Run `balena login`, select `Credentials` and use `SUPERUSER_EMAIL` and
`SUPERUSER_PASSWORD` generated during `make up` step to login to the openBalena server.
At any time, `balena whoami` command may be used to check which server the CLI is
authenticated with.
#### Create an application
Create a new application with `balena app create myApp`. Select the application's
Create a new application with `balena fleet create myApp`. Select the application's
default device type with the interactive prompt. The examples in this guide assume
a Raspberry Pi 3.
An application contains devices that share the same architecture (such as ARM
or Intel i386), and also contains code releases that are deployed to the devices.
When a device is provisioned, it is added to an application, but can be migrated
to another application at any time. There is no limit to the number of applications
that can be created or to the number of devices that can be provisioned.
An application contains devices that share the same architecture (such as ARM or Intel),
and also contains code releases that are deployed to the devices. When a device is
provisioned, it is added to an application, but can be migrated to another application at
any time. There is no limit to the number of applications that can be created or to the
number of devices that can be provisioned.
At any time, the server can be queried for all the applications it knows about
with the following command:
```bash
balena apps
ID APP NAME DEVICE TYPE ONLINE DEVICES DEVICE COUNT
1 myApp raspberrypi3
balena fleets
Id App name Slug Device type Device count Online devices
── ──────── ─────────── ──────────── ──────────── ──────────────
1 myApp admin/myapp raspberrypi3 0 0
```
#### Provision a new device
Once we have an application, it’s time to start provisioning devices. To do this,
first download a balenaOS image from [balena.io](https://balena.io/os/#download).
Pick the development image that is appropriate for your device.
first download a [balenaOS] image for your device. For this example we are using a
Raspberry Pi 3.
Unzip the downloaded image and use the balena CLI to configure it:
```bash
balena os configure ~/Downloads/balena-cloud-raspberrypi3-2.58.3+rev1-dev-v11.14.0.img --app myApp
balena os configure --dev --fleet myApp ~/Downloads/raspberrypi3-5.2.8-v16.1.10.img
```
Flash the configured image to an SD card using [Etcher](https://balena.io/etcher).
Insert the SD card into the device and power it on. The device will register with
the openBalena server and after about two minutes will be inspectable:
Flash the configured image to an SD card using [Etcher] or balena CLI:
```bash
sudo balena local flash ~/Downloads/raspberrypi3-5.2.8-v16.1.10.img
```
Insert the SD card into the device and power it on. The device will register with the
openBalena server and after about two minutes will be inspectable:
```bash
balena devices
ID UUID DEVICE NAME DEVICE TYPE APPLICATION NAME STATUS IS ONLINE SUPERVISOR VERSION OS VERSION
4 59d7700 winter-tree raspberrypi3 myApp Idle true 11.14.0 balenaOS 2.58.3+rev1
ID UUID DEVICE NAME DEVICE TYPE FLEET STATUS IS ONLINE SUPERVISOR VERSION OS VERSION
1 560dcc2 quiet-rock raspberrypi3 admin/myapp Idle true 16.1.10 balenaOS 5.2.8
balena device 59d7700
== WINTER TREE
ID: 4
balena device 560dcc2
== WANDERING RAIN
ID: 1
DEVICE TYPE: raspberrypi3
STATUS: online
STATUS: idle
IS ONLINE: true
IP ADDRESS: 192.168.43.247
APPLICATION NAME: myApp
UUID: 59d7700755ec5de06783eda8034c9d3d
SUPERVISOR VERSION: 11.14.0
OS VERSION: balenaOS 2.58.3+rev1
IP ADDRESS: 192.168.1.42
MAC ADDRESS: B8:27:DE:AD:BE:EF
FLEET: admin/myapp
LAST SEEN: 1977-08-20T14:29:00.042Z
UUID: 560dcc24b221c8a264d5bd981284801f
COMMIT: N/a
SUPERVISOR VERSION: 16.1.10
IS WEB ACCESSIBLE: false
OS VERSION: balenaOS 5.2.8
DASHBOARD URL: https://dashboard.mydomain.com/devices/560dcc24b221c8a264d5bd981284801f/summary
CPU USAGE PERCENT: 2
CPU TEMP C: 39
CPU ID: 00000000335956af
MEMORY USAGE MB: 140
MEMORY TOTAL MB: 971
MEMORY USAGE PERCENT: 14
STORAGE BLOCK DEVICE: /dev/mmcblk0p6
STORAGE USAGE MB: 76
STORAGE TOTAL MB: 14121
STORAGE USAGE PERCENT: 1
```
Note, even though the dashboard URL is populated, there is no dashboard service in
openBalena.
It's time to deploy code to the device.
#### Deploy a project
Application release images are built on the local machine using the balena CLI.
Ensure the root certificate has been correctly installed on the local machine,
as discussed above.
Application release images are built on the local machine using the balena CLI. Ensure the
root certificate has been correctly installed on the local machine, as discussed above.
Let's create a trivial project that logs "Idling...". On an empty directory,
create a new file named `Dockerfile.template` with the following contents:
Let's create a trivial project that logs "Idling...". On an empty directory, create a new
file named `Dockerfile.template` with the following contents:
```dockerfile
FROM balenalib/%%BALENA_MACHINE_NAME%%-alpine
@ -292,20 +384,21 @@ CMD [ "balena-idle" ]
Then build and deploy the project with:
```bash
balena deploy myApp --logs
balena deploy --noparent-check myApp
```
The project will have been successfully built when a friendly unicorn appears in
the terminal:
The project will have been successfully built when a friendly unicorn appears in the
terminal:
```bash
[Info] Compose file detected
...
[Info] No "docker-compose.yml" file found at "~/open-balena/balena-idle"
[Info] Creating default composition with source: "~/open-balena/balena-idle"
[Info] Everything is up to date (use --build to force a rebuild)
[Info] Creating release...
[Info] Pushing images to registry...
[Info] Saving release...
[Success] Deploy succeeded!
[Success] Release: f62a74c220b92949ec78761c74366046
[Success] Release: 50be7bdb0ea6819c91a5dd7bcd7635ad
\
\
@ -332,35 +425,61 @@ the terminal:
<'
```
This command packages up the local directory, creates a new Docker image from
it and pushes it to the openBalena server. In turn, the server will deploy it to
all provisioned devices and within a couple of minutes, they will all run the
new release. Logs can be viewed with:
This command packages up the local directory, creates a new Docker image from it and
pushes it to the openBalena server. In turn, the server will deploy it to all provisioned
devices and within a couple of minutes, they will all run the new release. Logs can be
viewed with:
```bash
balena logs 59d7700 --tail
[Logs] [10/28/2020, 11:40:16 AM] Supervisor starting
[Logs] [10/28/2020, 11:40:50 AM] Creating network 'default'
[Logs] [10/28/2020, 11:42:38 AM] Creating volume 'resin-data'
[Logs] [10/28/2020, 11:42:40 AM] Downloading image …
balena logs --tail 560dcc2
[Logs] [2024-05-02T15:59:31.383Z] Supervisor starting
[Logs] [2024-05-02T15:59:37.552Z] Applying configuration change {"SUPERVISOR_VPN_CONTROL":"true"}
[Logs] [2024-05-02T15:59:37.599Z] Applied configuration change {"SUPERVISOR_VPN_CONTROL":"true"}
[Logs] [2024-05-02T15:59:40.331Z] Creating network 'default'
[Logs] [2024-05-02T16:11:15.331Z] Supervisor starting
[Logs] [2024-05-02T16:44:08.199Z] Creating volume 'resin-data'
[Logs] [2024-05-02T16:44:08.572Z] Downloading image 'registry2.mydomain.com/v2/…
…
[Logs] [10/28/2020, 11:44:00 AM] [main] Idling...
[Logs] [2024-05-02T16:44:37.200Z] [main] Idling...
[Logs] [2024-05-02T16:44:37.200Z] [main] Idling...
```
Enjoy Balenafying All the Things!
## Next steps
- Try out [local mode](https://www.balena.io/docs/learn/develop/local-mode),
which allows you to build and sync code to your device locally for rapid
development.
- Develop an application with [multiple containers](https://www.balena.io/docs/learn/develop/multicontainer)
to provide a more modular approach to application management.
- Manage your device fleet with the use of [configuration](https://www.balena.io/docs/learn/manage/configuration/)
and [environment](https://www.balena.io/docs/learn/manage/serv-vars/) variables.
- Explore our [example projects](https://balena.io/blog/tags/etcher-featured/)
to give you an idea of more things you can do with balena.
- If you find yourself stuck or confused, help is just [a click away](https://www.balena.io/support).
- Pin selected devices to selected code releases using
[sample scripts](https://github.com/balena-io-examples/staged-releases).
- To change the superuser password after setting the credentials, follow this [forum post](https://forums.balena.io/t/upate-superuser-password/4738/6).
- Try out [local mode], which allows you to build and sync code to your device locally for
rapid development.
- Develop an application with [multiple containers] to provide a more modular approach to
application management.
- Manage your device fleet with the use of [configuration] and [environment] variables.
- Explore our [example projects] to give you an idea of more things you can do with
balena.
- If you find yourself stuck or confused, help is just [a click away].
- Pin selected devices to selected code releases using [sample scripts].
- To change the superuser password after setting the credentials, follow this [forum post]
[^1]: If DNS validation is not an option, [acme.sh] or [certbot] can be used to manually
issue a certificate, which can then be set using the [custom SSL](#custom-ssl) workflow.
[local mode]: https://www.balena.io/docs/learn/develop/local-mode
[multiple containers]: https://www.balena.io/docs/learn/develop/multicontainer
[configuration]: https://www.balena.io/docs/learn/manage/configuration
[environment]: https://www.balena.io/docs/learn/manage/serv-vars
[example projects]: https://balena.io/blog/tags/etcher-featured
[a click away]: https://www.balena.io/support
[sample scripts]: https://github.com/balena-io-examples/staged-releases
[forum post]: https://forums.balena.io/t/upate-superuser-password/4738/6
[balena CLI installation instructions]: https://github.com/balena-io/balena-cli/blob/master/INSTALL.md
[Etcher]: https://balena.io/etcher
[balenaOS]: https://balena.io/os/#download
[balenaEngine]: https://www.balena.io/engine
[Docker]: https://docs.docker.com/get-docker
[Docker Engine]: https://docs.docker.com/engine/install
[Change cgroup version]: https://docs.docker.com/config/containers/runmetrics/#changing-cgroup-version
[composition]: https://github.com/balena-io/open-balena/blob/master/docker-compose.yml
[DNS-01]: https://letsencrypt.org/docs/challenge-types/#dns-01-challenge
[acme.sh]: https://github.com/acmesh-official/acme.sh
[certbot]: https://certbot.eff.org/

6
repo.yml
View File

@ -1,5 +1,5 @@
type: "generic"
reviewers: 1
---
type: generic
upstream:
- repo: open-balena-api
url: https://github.com/balena-io/open-balena-api
@ -11,5 +11,3 @@ upstream:
url: https://github.com/balena-io/open-balena-db
- repo: open-balena-s3
url: https://github.com/balena-io/open-balena-s3
- repo: balena-mdns-publisher
url: https://github.com/balena-io/balena-mdns-publisher

79
scripts/_keyid.js
View File

@ -1,79 +0,0 @@
'use strict';
var crypto = require('crypto');
var fs = require('fs');
var base32 = (function() {
// Extracted from https://github.com/chrisumbel/thirty-two
// to avoid having to install packages for this script.
var charTable = "ABCDEFGHIJKLMNOPQRSTUVWXYZ234567";
var byteTable = [
0xff, 0xff, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f,
0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
0xff, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06,
0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e,
0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16,
0x17, 0x18, 0x19, 0xff, 0xff, 0xff, 0xff, 0xff,
0xff, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06,
0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e,
0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16,
0x17, 0x18, 0x19, 0xff, 0xff, 0xff, 0xff, 0xff
];
function quintetCount(buff) {
var quintets = Math.floor(buff.length / 5);
return buff.length % 5 == 0 ? quintets: quintets + 1;
}
return function(plain) {
if (!Buffer.isBuffer(plain)) {
plain = new Buffer(plain);
}
var i = 0;
var j = 0;
var shiftIndex = 0;
var digit = 0;
var encoded = new Buffer(quintetCount(plain) * 8);
/* byte by byte isn't as pretty as quintet by quintet but tests a bit
faster. will have to revisit. */
while(i < plain.length) {
var current = plain[i];
if(shiftIndex > 3) {
digit = current & (0xff >> shiftIndex);
shiftIndex = (shiftIndex + 5) % 8;
digit = (digit << shiftIndex) | ((i + 1 < plain.length) ?
plain[i + 1] : 0) >> (8 - shiftIndex);
i++;
} else {
digit = (current >> (8 - (shiftIndex + 5))) & 0x1f;
shiftIndex = (shiftIndex + 5) % 8;
if(shiftIndex == 0) i++;
}
encoded[j] = charTable.charCodeAt(digit);
j++;
}
for (i = j; i < encoded.length; i++) {
encoded[i] = 0x3d; //'='.charCodeAt(0)
}
return encoded;
}
})();
function joseKeyId(der) {
var hasher = crypto.createHash('sha256');
hasher.update(der);
var b32 = base32(hasher.digest().slice(0, 30)).toString('ascii');
var chunks = [];
for (var i = 0; i < b32.length; i += 4) {
chunks.push(b32.substr(i, 4));
}
return chunks.join(':');
}
var derFilePath = process.argv[2];
var der = fs.readFileSync(derFilePath);
process.stdout.write(joseKeyId(der));

35
scripts/_realpath
View File

@ -1,35 +0,0 @@
#!/bin/bash -e
echo_error() {
local RED=`tput setaf 1`
local RESET=`tput sgr0`
echo "${RED}ERROR: ${1}${RESET}"
}
REALPATH=
REALPATHS=(
'realpath'
'grealpath'
'greadlink -f'
)
for cmd in "${REALPATHS[@]}"; do
if command -v "${cmd%% *}" &>/dev/null; then
REALPATH="${cmd}"
break
fi
done
if [ -z "${REALPATH}" ]; then
echo_error 'Unable to find suitable command for realpath.'
if [ $(uname) == 'Darwin' ]; then
echo 'GNU coreutils are required to build openBalena on macOS. To install with brew, run'
echo ''
echo ' brew install coreutils'
echo ''
fi
exit 1
fi
realpath() {
echo $(command ${REALPATH} "$@")
}

40
scripts/compose
View File

@ -1,40 +0,0 @@
#!/bin/bash -e
source "${BASH_SOURCE%/*}/_realpath"
CMD="$(realpath "$0")"
DIR="$(dirname "${CMD}")"
BASE_DIR="$(dirname "${DIR}")"
CONFIG_DIR="${BASE_DIR}/config"
echo_bold() {
printf "\\033[1m%s\\033[0m\\n" "$@"
}
VERSIONS_FILE="${BASE_DIR}/compose/versions"
if [ ! -f "$VERSIONS_FILE" ]; then
echo_bold "No service versions defined in ${VERSIONS_FILE}"
exit 1
fi
ENV_FILE="${CONFIG_DIR}/activate"
if [ ! -f "$ENV_FILE" ]; then
echo_bold 'No configuration found; please create one first with: ./scripts/quickstart'
echo_bold 'See README.md for help.'
exit 1
fi
source "${ENV_FILE}"
# only include the MDNS publisher IF the domain is valid...
if [ ${OPENBALENA_HOST_NAME: -6} == ".local" ]; then
INCLUDE_MDNS="-f ${BASE_DIR}/compose/mdns.yml"
fi
# shellcheck source=/dev/null
source "${VERSIONS_FILE}"; docker-compose \
--project-name 'openbalena' \
-f "${BASE_DIR}/compose/services.yml" \
${INCLUDE_MDNS} \
-f "${CONFIG_DIR}/docker-compose.yml" \
"$@"

36
scripts/create-superuser
View File

@ -1,36 +0,0 @@
#!/bin/bash -e
usage() {
echo "usage: $0 EMAIL PASSWORD"
echo
echo 'Create the superuser account with the given email and password.'
echo
echo 'The instance must already be running in the background. You can '
echo 'start it with: ./scripts/compose up -d'
}
if [[ -z "$1" || -z "$2" ]]; then
usage
exit 1
fi
echo_bold() {
printf "\\033[1m%s\\033[0m\\n" "${@}"
}
source "${BASH_SOURCE%/*}/_realpath"
CMD="$(realpath "$0")"
DIR="$(dirname "${CMD}")"
FIG="${DIR}/compose"
EMAIL="$1"
PASSWORD="$2"
"${FIG}" exec api /bin/bash -c \
'export $(grep -v "^#" config/env | xargs -d "\n"); node index.js create-superuser root '${EMAIL}' '${PASSWORD}'' \
>/dev/null \
|| (echo 'Failed to create superuser; please ensure the instance is running and that no superuser has been created before.' && exit 1)
echo_bold "==> Success! Superuser created with email: ${EMAIL}"
echo " - You may now login with: balena login --credentials --email ${EMAIL}"

35
scripts/gen-root-ca
View File

@ -1,35 +0,0 @@
#!/bin/bash -e
usage() {
echo "usage: $0 COMMON_NAME [OUT]"
echo
echo " COMMON_NAME the domain name the certificate is valid for, eg. example.com"
echo " OUT path to output directory generated files will be placed in"
echo
}
if [ -z "$1" ]; then
usage
exit 1
fi
CMD="$(realpath "$0")"
DIR="$(dirname "${CMD}")"
CN="$1"
OUT="$(realpath "${2:-.}")"
# shellcheck source=scripts/ssl-common.sh
source "${DIR}/ssl-common.sh"
ROOT_CA="${ROOT_PKI}/ca.crt"
if [ ! -f $ROOT_CA ]; then
# Create a secret key and CA file for the self-signed CA
"$easyrsa_bin" --pki-dir="${ROOT_PKI}" init-pki 2>/dev/null
"$easyrsa_bin" --pki-dir="${ROOT_PKI}" --days="${CA_EXPIRY_DAYS}" --req-cn="ca.${CN}" build-ca nopass 2>/dev/null
# update indexes and generate CRLs
"$easyrsa_bin" --pki-dir="${ROOT_PKI}" update-db 2>/dev/null
"$easyrsa_bin" --pki-dir="${ROOT_PKI}" gen-crl 2>/dev/null
fi

36
scripts/gen-root-cert
View File

@ -1,36 +0,0 @@
#!/bin/bash -e
usage() {
echo "usage: $0 COMMON_NAME [OUT]"
echo
echo " COMMON_NAME the domain name the certificate is valid for, eg. example.com"
echo " OUT path to output directory generated files will be placed in"
echo
}
if [ -z "$1" ]; then
usage
exit 1
fi
CMD="$(realpath "$0")"
DIR="$(dirname "${CMD}")"
CN="$1"
OUT="$(realpath "${2:-.}")"
# shellcheck source=scripts/ssl-common.sh
source "${DIR}/ssl-common.sh"
ROOT_CRT="${ROOT_PKI}"'/issued/*.'"${CN}"'.crt'
ROOT_KEY="${ROOT_PKI}"'/private/*.'"${CN}"'.key'
if [ ! -f $ROOT_CRT ] || [ ! -f $ROOT_KEY ]; then
rm -f $ROOT_CRT $ROOT_KEY
# generate default CSR and sign (root + wildcard)
"$easyrsa_bin" --pki-dir="${ROOT_PKI}" --days="${CRT_EXPIRY_DAYS}" --subject-alt-name="DNS:*.${CN}" build-server-full "*.${CN}" nopass 2>/dev/null
# update indexes and generate CRLs
"$easyrsa_bin" --pki-dir="${ROOT_PKI}" update-db 2>/dev/null
"$easyrsa_bin" --pki-dir="${ROOT_PKI}" gen-crl 2>/dev/null
fi;

53
scripts/gen-token-auth-cert
View File

@ -1,53 +0,0 @@
#!/bin/bash -e
usage() {
echo "usage: $0 COMMON_NAME [OUT]"
echo
echo " COMMON_NAME the domain name the certificate is valid for, eg. example.com"
echo " OUT path to output directory generated files will be placed in"
echo
}
if [ -z "$1" ]; then
usage
exit 1
fi
CMD="$(realpath "$0")"
DIR="$(dirname "${CMD}")"
CN="$1"
OUT="$(realpath "${2:-.}")"
# shellcheck source=scripts/ssl-common.sh
source "${DIR}/ssl-common.sh"
CERT_DIR="${OUT}/api"
CERT_FILE="${CERT_DIR}/api.${CN}"
keyid() {
# NodeJS is installed as `nodejs` in some distros, `node` in others.
node_bin="$(command -v nodejs 2>/dev/null || command -v node 2>/dev/null || true)"
if [ -z "$node_bin" ]; then
echo >&2 'NodeJS is required but not installed. Aborting.'
exit 1
fi
# Recent Node versions complain about `new Buffer()` being deprecated
# but the alternative is not available to older versions. Silence the
# warning but use the deprecated form to allow greater compatibility.
"$node_bin" --no-deprecation "${DIR}/_keyid.js" "$1"
}
JWT_CRT="${CERT_FILE}.crt"
JWT_KEY="${CERT_FILE}.pem"
JWT_KID="${CERT_FILE}.kid"
if [ ! -f $JWT_CRT ] || [ ! -f $JWT_KEY ] || [ ! -f $JWT_KID ]; then
rm -f $JWT_CRT $JWT_KEY $JWT_KID
mkdir -p "${CERT_DIR}"
openssl ecparam -name prime256v1 -genkey -noout -out "${JWT_KEY}" 2>/dev/null
openssl req -x509 -new -nodes -days "${CRT_EXPIRY_DAYS}" -key "${JWT_KEY}" -subj "/CN=api.${CN}" -out "${JWT_CRT}" 2>/dev/null
openssl ec -in "${JWT_KEY}" -pubout -outform DER -out "${CERT_FILE}.der" 2>/dev/null
keyid "${CERT_FILE}.der" >"${JWT_KID}"
rm "${CERT_FILE}.der"
fi

48
scripts/gen-vpn-certs
View File

@ -1,48 +0,0 @@
#!/bin/bash -e
usage() {
echo "usage: $0 COMMON_NAME [OUT]"
echo
echo " COMMON_NAME the domain name the certificate is valid for, eg. example.com"
echo " OUT path to output directory generated files will be placed in"
echo
}
if [ -z "$1" ]; then
usage
exit 1
fi
CMD="$(realpath "$0")"
DIR="$(dirname "${CMD}")"
CN="$1"
OUT="$(realpath "${2:-.}")"
# shellcheck source=scripts/ssl-common.sh
source "${DIR}/ssl-common.sh"
VPN_PKI="$(realpath "${OUT}/vpn")"
VPN_CA="${VPN_PKI}/ca.crt"
VPN_CRT="${VPN_PKI}/issued/vpn.${CN}.crt"
VPN_KEY="${VPN_PKI}/private/vpn.${CN}.key"
VPN_DH="${VPN_PKI}/dh.pem"
if [ ! -f $VPN_CA ] || [ ! -f $VPN_CRT ] || [ ! -f $VPN_KEY ] || [ ! -f $VPN_DH ]; then
rm -f $VPN_CA $VPN_CRT $VPN_DH $VPN_KEY
# generate VPN CA
"$easyrsa_bin" --pki-dir="${VPN_PKI}" init-pki &>/dev/null
"$easyrsa_bin" --pki-dir="${VPN_PKI}" --days="${CA_EXPIRY_DAYS}" --req-cn="vpn-ca.${CN}" build-ca nopass 2>/dev/null
# generate and sign vpn server certificate
"$easyrsa_bin" --pki-dir="${VPN_PKI}" --days="${CRT_EXPIRY_DAYS}" build-server-full "vpn.${CN}" nopass 2>/dev/null
# generate vpn dhparams (keysize of 2048 will do, 4096 can wind up taking hours to generate)
"$easyrsa_bin" --pki-dir="${VPN_PKI}" --keysize=2048 gen-dh 2>/dev/null
# update indexes and generate CRLs
"$easyrsa_bin" --pki-dir="${VPN_PKI}" update-db 2>/dev/null
"$easyrsa_bin" --pki-dir="${VPN_PKI}" gen-crl 2>/dev/null
fi

62
scripts/logger.sh
View File

@ -1,62 +0,0 @@
#!/bin/sh
BLACK=`tput setaf 0`
RED=`tput setaf 1`
GREEN=`tput setaf 2`
YELLOW=`tput setaf 3`
BLUE=`tput setaf 4`
MAGENTA=`tput setaf 5`
CYAN=`tput setaf 6`
WHITE=`tput setaf 7`
BOLD=`tput bold`
RESET=`tput sgr0`
log_raw () {
local COLOR="${WHITE}"
local LEVEL="${1}"
local MESSAGE="${2}"
case "${LEVEL}" in
info)
COLOR="${BLUE}"
;;
warn)
COLOR="${YELLOW}"
;;
fatal)
COLOR="${RED}"
;;
*)
LEVEL="debug"
;;
esac
LEVEL="${LEVEL} "
echo "[$(date +%T)] ${COLOR}$(echo "${LEVEL:0:5}" | tr '[:lower:]' '[:upper:]')${RESET} ${MESSAGE}";
}
log () {
log_raw "debug" "${1}"
}
info () {
log_raw "info" "${1}";
}
warn () {
log_raw "warn" "${1}";
}
die () {
log_raw "fatal" "${1}";
exit 1;
}
die_unless_forced () {
if [ ! -z "$1" ]; then
log_raw "warn" "$2";
return;
fi
log_raw "fatal" "$2";
die "Use -f to forcibly upgrade.";
}

77
scripts/make-env
View File

@ -1,77 +0,0 @@
#!/bin/bash -e
usage() {
echo "usage: $0"
echo
echo "Required Variables:"
echo
echo " DOMAIN"
echo " ROOT_CA Path to root CA certificate"
echo " ROOT_CRT Path to root/wildcard certificate"
echo " ROOT_KEY Path to root/wildcard private key"
echo " JWT_CRT Path to Token Auth certificate"
echo " JWT_KEY Path to Token Auth private key"
echo " JWT_KID Path to KeyID for the Token Auth certificate"
echo " VPN_CA Path to the VPN CA certificate"
echo " VPN_CRT Path to the VPN server certificate"
echo " VPN_KEY Path to the VPN server private key"
echo " VPN_DH Path to the VPN server Diffie Hellman parameters"
echo " SUPERUSER_EMAIL Email address of the superuser"
echo " SUPERUSER_PASSWORD Password of the superuser"
echo
}
for var in DOMAIN ROOT_CA ROOT_CRT ROOT_KEY JWT_CRT JWT_KEY JWT_KID VPN_CA VPN_CRT VPN_KEY VPN_DH SUPERUSER_EMAIL SUPERUSER_PASSWORD; do
if [ -z "${!var-}" ]; then
usage
exit 1
fi
done
randstr() {
LC_CTYPE=C tr -dc A-Za-z0-9 < /dev/urandom | fold -w "${1:-32}" | head -n 1
}
b64encode() {
echo "$@" | base64 --wrap=0 2>/dev/null || echo "$@" | base64 --break=0 2>/dev/null
}
b64file() {
b64encode "$(cat "$@")"
}
# buckets to create in the S3 service...
REGISTRY2_S3_BUCKET="registry-data"
cat <<STR
export OPENBALENA_PRODUCTION_MODE=false
export OPENBALENA_COOKIE_SESSION_SECRET=$(randstr 32)
export OPENBALENA_HOST_NAME=$DOMAIN
export OPENBALENA_JWT_SECRET=$(randstr 32)
export OPENBALENA_REGISTRY2_S3_BUCKET=${REGISTRY2_S3_BUCKET}
export OPENBALENA_RESINOS_REGISTRY_CODE=$(randstr 32)
export OPENBALENA_ROOT_CA=$(b64file "${ROOT_CA}")
export OPENBALENA_ROOT_CRT=$(b64file "${ROOT_CRT}")
export OPENBALENA_ROOT_KEY=$(b64file "${ROOT_KEY}")
export OPENBALENA_TOKEN_AUTH_BUILDER_TOKEN=$(randstr 64)
export OPENBALENA_TOKEN_AUTH_PUB=$(b64file "$JWT_CRT")
export OPENBALENA_TOKEN_AUTH_KEY=$(b64file "$JWT_KEY")
export OPENBALENA_TOKEN_AUTH_KID=$(b64file "$JWT_KID")
export OPENBALENA_VPN_CA=$(b64file "$VPN_CA")
export OPENBALENA_VPN_CA_CHAIN=$(b64file "$VPN_CA")
export OPENBALENA_VPN_SERVER_CRT=$(b64file "$VPN_CRT")
export OPENBALENA_VPN_SERVER_KEY=$(b64file "$VPN_KEY")
export OPENBALENA_VPN_SERVER_DH=$(b64file "$VPN_DH")
export OPENBALENA_VPN_SERVICE_API_KEY=$(randstr 32)
export OPENBALENA_API_VPN_SERVICE_API_KEY=$(randstr 32)
export OPENBALENA_REGISTRY_SECRET_KEY=$(randstr 32)
export OPENBALENA_S3_ACCESS_KEY=$(randstr 32)
export OPENBALENA_S3_BUCKETS="${REGISTRY2_S3_BUCKET}"
export OPENBALENA_S3_ENDPOINT="https://s3.${DOMAIN}"
export OPENBALENA_S3_REGION=us-east-1
export OPENBALENA_S3_SECRET_KEY=$(randstr 32)
export OPENBALENA_SSH_AUTHORIZED_KEYS=
export OPENBALENA_SUPERUSER_EMAIL=$SUPERUSER_EMAIL
export OPENBALENA_SUPERUSER_PASSWORD=$(printf "%q" "${SUPERUSER_PASSWORD}")
export OPENBALENA_ACME_CERT_ENABLED=${ACME_CERT_ENABLED:-false}
STR

29
scripts/migrate-registry-storage
View File

@ -1,29 +0,0 @@
#!/bin/sh
migrate_data_to_s3 () {
BUCKET="${1:-registry-data}"
if [ -z "${BUCKET}" ]; then return 1; fi
if [ -n "${DOCKER_HOST}" ]; then
log "Using docker host: ${DOCKER_HOST}"
export DOCKER_HOST="${DOCKER_HOST}"
fi
REGISTRY_CONTAINER="$(docker ps | grep registry_ | awk '{print $1}')"
S3_CONTAINER="$(docker ps | grep s3_ | awk '{print $1}')"
if [ -z "${REGISTRY_CONTAINER}" ] || [ -z "${S3_CONTAINER}" ]; then return 2; fi
REGISTRY_VOLUME="$(docker inspect "${REGISTRY_CONTAINER}" | jq -r '.[].Mounts | map(select(.Destination=="/data")) | .[0].Source')"
S3_VOLUME=$(docker inspect "${S3_CONTAINER}" | jq -r '.[].Mounts | map(select(.Destination=="/export")) | .[0].Source')
if [ -z "${REGISTRY_VOLUME}" ] || [ -z "${S3_VOLUME}" ]; then return 3; fi
# run the S3 container image, and copy the data partition into S3...
docker run -it --rm \
-v "${REGISTRY_VOLUME}:/data" \
-v "${S3_VOLUME}:/s3" \
--name "migrate-registry" alpine \
sh -c "mkdir -p /s3/${BUCKET}/data && cp -r /data/docker /s3/${BUCKET}/data/"
}

30
scripts/patch-hosts
View File

@ -1,30 +0,0 @@
#!/bin/bash -e
usage() {
echo "usage: $0 DOMAIN"
echo
echo " DOMAIN the domain name to add host entries for, eg. example.com"
echo
}
if [ -z "$1" ]; then
usage
exit 1
fi
SERVICES="api registry vpn db s3 redis"
DOMAIN="$1"
# We need sudo to write to /etc/hosts, so first write to a temp file and then
# append all entries to hosts file.
tmp="$(mktemp --tmpdir openbalena.XXXX)"
for service in $SERVICES; do
name="${service}.${DOMAIN}"
if ! grep "\\s$name" /etc/hosts >/dev/null 2>&1 ; then
echo "adding $name"
echo "127.0.0.1 $name" >>"${tmp}"
fi
done
# shellcheck disable=SC2024
sudo tee -a /etc/hosts >/dev/null <"${tmp}"
rm -f "${tmp}"

155
scripts/quickstart
View File

@ -1,155 +0,0 @@
#!/bin/bash -e
BLACK=`tput setaf 0`
RED=`tput setaf 1`
GREEN=`tput setaf 2`
YELLOW=`tput setaf 3`
BLUE=`tput setaf 4`
MAGENTA=`tput setaf 5`
CYAN=`tput setaf 6`
WHITE=`tput setaf 7`
BOLD=`tput bold`
RESET=`tput sgr0`
# for macos machines, we need proper OpenSSL...
OPENSSL_VERSION=$(openssl version -v)
if [[ "${OPENSSL_VERSION}" =~ ^LibreSSL.*$ ]]; then
echo -e "${RED}ERROR: You may not have a compatible OpenSSL version (${OPENSSL_VERSION}). Please install OpenSSL version 1.0.2q or above.${RESET}"
if [ $(uname) == 'Darwin' ]; then
echo 'OpenSSL is required to build openBalena on macOS. To install with brew, run'
echo ''
echo ' brew install openssl'
echo ''
fi
exit 1
fi
source "${BASH_SOURCE%/*}/_realpath"
domainResolves() {
getent hosts "$1" > /dev/null 2>&1
}
CMD="$(realpath "$0")"
DIR="$(dirname "${CMD}")"
BASE_DIR="$(dirname "${DIR}")"
CONFIG_DIR="${BASE_DIR}/config"
CERTS_DIR="${CONFIG_DIR}/certs"
DOMAIN=openbalena.local
usage() {
echo "usage: $0 [-c] [-h] [-p] [-d DOMAIN] -U EMAIL -P PASSWORD"
echo
echo " -c enable the ACME certificate service in staging or production mode."
echo " -p patch hosts - patch the host /etc/hosts file"
echo " -d DOMAIN the domain name this deployment will run as, eg. example.com. Default is 'openbalena.local'"
echo " -U EMAIL the email address of the superuser account, used to login to your install from the Balena CLI"
echo " -P PASSWORD the password to use for the superuser account."
echo
}
show_help=false
patch_hosts=false
while getopts ":chpxd:U:P:" opt; do
case "${opt}" in
h) show_help=true;;
p) patch_hosts=true;;
x) set -x;;
d) DOMAIN="${OPTARG}";;
U) SUPERUSER_EMAIL="${OPTARG}";;
P) SUPERUSER_PASSWORD="${OPTARG}";;
c) ACME_CERT_ENABLED="true";;
*)
echo "Invalid argument: -${OPTARG}"
usage
exit 1
;;
esac
done
shift $((OPTIND-1))
if [ -z "${SUPERUSER_EMAIL}" ] || [ -z "${SUPERUSER_PASSWORD}" ]; then
usage
exit 1
fi
if [ "$show_help" = "true" ]; then
usage
exit 1
fi
if [ ! -z "$ACME_CERT_ENABLED" ]; then
echo "${BLUE}[INFO]${RESET} ACME Certificate request is ${BOLD}ENABLED${RESET}."
if ! domainResolves "api.${DOMAIN}"; then
echo "${YELLOW}[WARN]${RESET} Unable to resolve \"api.${DOMAIN}\"!"
echo "${YELLOW}[WARN]${RESET} This might mean that you cannot use an ACME issued certificate."
fi
fi
echo_bold() {
echo "${BOLD}${@}${RESET}"
}
echo_bold "==> Creating new configuration at: $CONFIG_DIR"
mkdir -p "$CONFIG_DIR" "$CERTS_DIR"
echo_bold "==> Bootstrapping easy-rsa..."
source "${DIR}/ssl-common.sh"
echo_bold "==> Generating root CA cert..."
# shellcheck source=scripts/gen-root-ca
source "${DIR}/gen-root-ca" "${DOMAIN}" "${CERTS_DIR}"
echo_bold "==> Generating root cert chain for haproxy..."
# shellcheck source=scripts/gen-root-cert
source "${DIR}/gen-root-cert" "${DOMAIN}" "${CERTS_DIR}"
echo_bold "==> Generating token auth cert..."
# shellcheck source=scripts/gen-token-auth-cert
source "${DIR}/gen-token-auth-cert" "${DOMAIN}" "${CERTS_DIR}"
echo_bold "==> Generating VPN CA, cert and dhparam (this may take a while)..."
# shellcheck source=scripts/gen-vpn-certs
source "${DIR}/gen-vpn-certs" "${DOMAIN}" "${CERTS_DIR}"
echo_bold "==> Setting up environment..."
# shellcheck source=scripts/make-env
cat >"${CONFIG_DIR}/activate" <(source "${DIR}/make-env")
echo_bold "==> Adding default compose file..."
cp "${BASE_DIR}/compose/template.yml" "${CONFIG_DIR}/docker-compose.yml"
if [ "${patch_hosts}" = "true" ]; then
echo_bold "==> Patching /etc/hosts..."
# shellcheck source=scripts/patch-hosts
source "${DIR}/patch-hosts" "${DOMAIN}"
fi
echo_bold "==> Success!"
echo ' - Start the instance with: ./scripts/compose up -d'
echo ' - Stop the instance with: ./scripts/compose stop'
echo ' - To create a single, flat, docker-compose.yml file, run:'
echo ''
echo ' ./scripts/compose config > docker-compose.yml'
echo ''
if [ -z "${ACME_CERT_ENABLED}" ]; then
echo " - Use the following certificate with Balena CLI: ${CERTS_DIR}/root/ca.crt"
case $(uname) in
Darwin)
echo ''
printf ' On macOS:\n\n'
printf ' sudo security add-trusted-cert -d -r trustRoot -k "/Library/Keychains/System.keychain" "%s/root/ca.crt"\n' "${CERTS_DIR}"
echo ''
;;
*)
;;
esac
echo -e " ${YELLOW}IMPORTANT:${RESET} You will need to restart your Docker daemon after trusting this certificate to allow your workstation to push images to the registry."
echo ''
fi

25
scripts/ssl-common.sh
View File

@ -1,25 +0,0 @@
#!/bin/bash -e
# shellcheck disable=SC2034
# ensure we have `easyrsa` available
if [ -z "${easyrsa_bin-}" ] || [ ! -x "${easyrsa_bin}" ]; then
easyrsa_bin="$(command -v easyrsa 2>/dev/null || true)"
if [ -z "${easyrsa_bin}" ]; then
easyrsa_dir="$(mktemp -dt easyrsa.XXXXXXXX)"
easyrsa_url="https://github.com/OpenVPN/easy-rsa/releases/download/v3.1.3/EasyRSA-3.1.3.tgz"
echo " - Downloading easy-rsa..."
(cd "${easyrsa_dir}"; curl -sL "${easyrsa_url}" | tar xz --strip-components=1)
easyrsa_bin="${easyrsa_dir}/easyrsa"
# shellcheck disable=SC2064
trap "rm -rf \"${easyrsa_dir}\"" EXIT
fi
export EASYRSA_BATCH=1
export EASYRSA_KEY_SIZE=4096
fi
# setup ROOT_PKI path
ROOT_PKI="$(realpath "${OUT}/root")"
# global expiry settings
CA_EXPIRY_DAYS=3650
CRT_EXPIRY_DAYS=730

78
scripts/upgrade-1.x-to-2.0
View File

@ -1,78 +0,0 @@
#!/bin/sh
source "${BASH_SOURCE%/*}/logger.sh"
source "${BASH_SOURCE%/*}/migrate-registry-storage"
# This script takes a v1.x.x install and updates the compose stack to use S3 as your
# registry storage.
source "${BASH_SOURCE%/*}/_realpath"
DIR="$(dirname $(realpath "$0"))"
BASE_DIR="$(dirname "${DIR}")"
CONFIG_DIR="${BASE_DIR}/config"
CONFIG_FILE="${CONFIG_DIR}/activate"
# Step 1. Make sure a config exists...
[ -f "${CONFIG_FILE}" ] || die "Unable to find existing config!";
info "Preparing to upgrade..."
source "${CONFIG_FILE}"
while getopts "f" opt; do
case "${opt}" in
f)
warn "Forcing upgrade! I hope you know what you're doing..."
FORCE_UPGRADE=1
;;
*)
echo "Invalid argument: ${OPTARG}"
exit 1
;;
esac
done
shift $((OPTIND-1))
# Step 2. Check if the S3 configuration already exists...
upgrade_required () {
[ -z "${OPENBALENA_REGISTRY2_S3_BUCKET}" ] || return 1;
[ -z "${OPENBALENA_S3_ACCESS_KEY}" ] || return 1;
[ -z "${OPENBALENA_S3_ENDPOINT}" ] || return 1;
[ -z "${OPENBALENA_S3_REGION}" ] || return 1;
[ -z "${OPENBALENA_S3_SECRET_KEY}" ] || return 1;
}
upgrade_required || die_unless_forced "${FORCE_UPGRADE}" "Configuration may already be using S3 for Registry storage!"
# Step 3. Create missing S3 configuration...
randstr() {
LC_CTYPE=C tr -dc A-Za-z0-9 < /dev/urandom | fold -w "${1:-32}" | head -n 1
}
upsert_config () {
var="${1}"
value="${2}"
if [ -z "${!var}" ]; then
echo "export ${1}=${2}" >> "${CONFIG_FILE}"
else
sed -i '' "s~export ${1}=.*~export ${1}=${2}~" "${CONFIG_FILE}"
fi
}
upsert_config "OPENBALENA_REGISTRY2_S3_BUCKET" "registry-data" || warn "Failed to update config value OPENBALENA_REGISTRY2_S3_BUCKET"
upsert_config "OPENBALENA_S3_ACCESS_KEY" "$(randstr 32)" || warn "Failed to update config value OPENBALENA_S3_ACCESS_KEY"
upsert_config "OPENBALENA_S3_ENDPOINT" "https://s3.${OPENBALENA_HOST_NAME}" || warn "Failed to update config value OPENBALENA_S3_ENDPOINT"
upsert_config "OPENBALENA_S3_REGION" "us-east-1" || warn "Failed to update config value OPENBALENA_S3_REGION"
upsert_config "OPENBALENA_S3_SECRET_KEY" "$(randstr 32)" || warn "Failed to update config value OPENBALENA_S3_SECRET_KEY"
# Step 4. Migrate Registry data to S3...
info "Copying data from the Registry volume to the S3 volume..."
migrate_data_to_s3 "registry-data"
case $? in
1) die "Invalid bucket name";;
2) die "Unable to find the running Registry or S3 containers";;
3) die "Unable to determine the data volumes for the Registry or S3 containers";;
*) info "Registry data copied"
;;
esac
info "Upgrade complete"

1
src/README
View File

@ -1 +0,0 @@
This is the working folder for any specific container you might want to work on.

34
src/balena-tests/Dockerfile Normal file
View File

@ -0,0 +1,34 @@
FROM ubuntu:22.04
# renovate: datasource=github-releases depName=balena-io/balena-cli
ARG BALENA_CLI_VERSION=v18.2.2
RUN apt-get update && apt-get install -y --no-install-recommends \
bash \
ca-certificates \
curl \
jq \
openssl \
procmail \
qemu-utils \
unzip \
wget \
&& rm -rf /var/lib/apt/lists/*
# https://docs.docker.com/engine/install/ubuntu/#install-using-the-convenience-script
RUN curl -fsSL https://get.docker.com | sh
WORKDIR /opt
RUN set -x; arch=$(uname -m | sed 's/86_64/64/g') \
&& wget -q "https://github.com/balena-io/balena-cli/releases/download/${BALENA_CLI_VERSION}/balena-cli-${BALENA_CLI_VERSION}-linux-${arch}-standalone.zip" \
&& unzip -q "balena-cli-${BALENA_CLI_VERSION}-linux-${arch}-standalone.zip" \
&& rm -rf "balena-cli-${BALENA_CLI_VERSION}-linux-${arch}-standalone.zip"
ENV PATH=/opt/balena-cli:${PATH}
COPY functions balena.sh /usr/sbin/
WORKDIR /balena
CMD /usr/sbin/balena.sh

328
src/balena-tests/balena.sh Executable file
View File

@ -0,0 +1,328 @@
#!/usr/bin/env bash
# shellcheck disable=SC2154,SC2034,SC1090
set -ae
curl_opts="--retry 3 --fail"
if [[ $VERBOSE =~ on|On|Yes|yes|true|True ]]; then
set -x
curl_opts="${curl_opts} --verbose"
else
curl_opts="${curl_opts} --silent"
fi
source /usr/sbin/functions
function remove_test_assets() {
rm -rf /balena/config.json \
"${GUEST_IMAGE}" \
"${GUEST_IMAGE%.*}.ready" \
"${tmpbuild}" \
/tmp/*.img
}
function remove_update_lock() {
rm -f /tmp/balena/updates.lock
}
function cleanup() {
shutdown_dut
remove_test_assets
remove_update_lock
# crash loop backoff
sleep "$(( (RANDOM % 5) + 5 ))s"
}
trap 'cleanup' EXIT
function shutdown_dut() {
local balena_device_uuid
balena_device_uuid="$(cat </balena/config.json | jq -r .uuid)"
if [[ -n $balena_device_uuid ]]; then
with_backoff balena device "${balena_device_uuid}"
balena device shutdown -f "${balena_device_uuid}" || true
fi
}
function set_update_lock {
if [[ -n "$BALENA_SUPERVISOR_ADDRESS" ]] && [[ -n "$BALENA_SUPERVISOR_API_KEY" ]]; then
while [[ $(curl ${curl_opts} "${BALENA_SUPERVISOR_ADDRESS}/v1/device?apikey=${BALENA_SUPERVISOR_API_KEY}" \
-H "Content-Type: application/json" | jq -r '.update_pending') == 'true' ]]; do
curl ${curl_opts} "${BALENA_SUPERVISOR_ADDRESS}/v1/device?apikey=${BALENA_SUPERVISOR_API_KEY}" \
-H "Content-Type: application/json" | jq -r
sleep "$(( (RANDOM % 3) + 3 ))s"
done
sleep "$(( (RANDOM % 5) + 5 ))s"
# https://www.balena.io/docs/learn/deploy/release-strategy/update-locking/
lockfile /tmp/balena/updates.lock
fi
}
function update_ca_certificates() {
# only set CA bundle if using private certificate chain
if [[ -e "${CERTS}/ca-bundle.pem" ]]; then
if [[ "$(readlink -f "${CERTS}/${TLD}-chain.pem")" =~ \/private\/ ]]; then
mkdir -p /usr/local/share/ca-certificates
cat <"${CERTS}/ca-bundle.pem" > /usr/local/share/ca-certificates/balenaRootCA.crt
# shellcheck disable=SC2034
CURL_CA_BUNDLE=${CURL_CA_BUNDLE:-${CERTS}/ca-bundle.pem}
NODE_EXTRA_CA_CERTS=${NODE_EXTRA_CA_CERTS:-${CURL_CA_BUNDLE}}
# (TBC) refactor to use NODE_EXTRA_CA_CERTS instead of ROOT_CA
# https://github.com/balena-io/e2e/blob/master/conf.js#L12-L14
# https://github.com/balena-io/e2e/blob/master/Dockerfile#L82-L83
# ... or
# https://thomas-leister.de/en/how-to-import-ca-root-certificate/
# https://github.com/puppeteer/puppeteer/issues/2377
ROOT_CA=${ROOT_CA:-$(cat <"${NODE_EXTRA_CA_CERTS}" | openssl base64 -A)}
else
rm -f /usr/local/share/ca-certificates/balenaRootCA.crt
unset NODE_EXTRA_CA_CERTS CURL_CA_BUNDLE ROOT_CA
fi
update-ca-certificates
fi
}
function wait_for_api() {
while ! curl ${curl_opts} "https://api.${DNS_TLD}/ping"; do
sleep "$(( (RANDOM % 5) + 5 ))s"
done
}
function open_balena_login() {
while ! balena login --credentials \
--email "${SUPERUSER_EMAIL}" \
--password "${SUPERUSER_PASSWORD}"; do
sleep "$(( (RANDOM % 5) + 5 ))s"
done
}
function create_fleet() {
if ! balena fleet "${TEST_FLEET}"; then
# wait for API to load DT contracts
while ! balena fleet create "${TEST_FLEET}" --type "${DEVICE_TYPE}"; do
sleep "$(( (RANDOM % 5) + 5 ))s"
done
# FIXME: on openBalena 'balena devices supported' always returns empty list
balena devices supported
fi
}
function download_os_image() {
if ! [[ -s "$GUEST_IMAGE" ]]; then
with_backoff wget -qO /tmp/balena.zip \
"${BALENA_API_URL}/download?deviceType=${DEVICE_TYPE}&version=${OS_VERSION:1}&fileType=.zip"
unzip -oq /tmp/balena.zip -d /tmp
cat <"$(find /tmp/ -type f -name '*.img' | head -n 1)" >"${GUEST_IMAGE}"
rm /tmp/balena.zip
fi
}
function configure_virtual_device() {
while ! [[ -s "$GUEST_IMAGE" ]]; do sleep "$(( (RANDOM % 5) + 5 ))s"; done
if ! [[ -s /balena/config.json ]]; then
balena_device_uuid="$(openssl rand -hex 16)"
with_backoff balena device register "${TEST_FLEET}" \
--uuid "${balena_device_uuid}"
with_backoff balena config generate \
--version "${OS_VERSION:1}" \
--device "${balena_device_uuid}" \
--network ethernet \
--appUpdatePollInterval 10 \
--dev \
--output /balena/config.json
fi
cat </balena/config.json | jq -re
with_backoff balena os configure "${GUEST_IMAGE}" \
--fleet "${TEST_FLEET}" \
--version "${OS_VERSION#v}" \
--config-network ethernet \
--config /balena/config.json
touch "${GUEST_IMAGE%.*}.ready"
}
function check_device_status() {
if [[ -e /balena/config.json ]]; then
balena_device_uuid="$(cat </balena/config.json | jq -r .uuid)"
if [[ -n $balena_device_uuid ]]; then
is_online="$(balena devices --json --fleet "${TEST_FLEET}" \
| jq -r --arg uuid "${balena_device_uuid}" '.[] | select(.uuid==$uuid).is_online == true')"
if [[ $is_online =~ true ]]; then
return 0
else
return 1
fi
fi
fi
}
function wait_for_device() {
while ! check_device_status; do sleep "$(( (RANDOM % 5) + 5 ))s"; done
}
function registry_auth() {
if [[ -n $REGISTRY_USER ]] && [[ -n $REGISTRY_PASS ]]; then
with_backoff docker login -u "${REGISTRY_USER}" -p "${REGISTRY_PASS}"
printf '{"https://index.docker.io/v1/": {"username":"%s", "password":"$s"}}' \
"${REGISTRY_USER}" "${REGISTRY_PASS}" | jq -r > ~/.balena/secrets.json
fi
}
function deploy_release() {
tmpbuild="$(mktemp -d)"
pushd "${tmpbuild}"
echo 'FROM hello-world' >Dockerfile
while ! balena deploy \
--ca "${DOCKER_CERT_PATH}/ca.pem" \
--cert "${DOCKER_CERT_PATH}/cert.pem" \
--key "${DOCKER_CERT_PATH}/key.pem" \
"${TEST_FLEET}"; do
sleep "$(( (RANDOM % 5) + 5 ))s"
done
popd
}
function get_releases() {
with_backoff balena releases --json "${TEST_FLEET}"
}
function get_release_commit() {
echo "$(get_releases)" | jq -re \
'select((.[].status=="success")
and (.[].is_invalidated==false)
and (.[].is_final==true)
and (.[].release_type=="final"))[0].commit'
}
function get_release_id() {
echo "$(get_releases)" | jq -re \
'select((.[].status=="success")
and (.[].is_invalidated==false)
and (.[].is_final==true)
and (.[].release_type=="final"))[0].id'
}
function supervisor_update_target_state() {
local balena_device_uuid
balena_device_uuid="$(cat </balena/config.json | jq -r .uuid)"
if [[ -n $balena_device_uuid ]]; then
while ! curl ${curl_opts} "https://api.${DNS_TLD}/supervisor/v1/update" \
--header "Content-Type: application/json" \
--header "Authorization: Bearer $(cat <~/.balena/token)" \
--data "{\"uuid\": \"${balena_device_uuid}\", \"data\": {\"force\": true}}"; do
sleep "$(( (RANDOM % 5) + 5 ))s"
done
fi
}
function check_running_release() {
local balena_device_uuid
balena_device_uuid="$(cat </balena/config.json | jq -r .uuid)"
local should_be_running_release
should_be_running_release="$(get_release_commit)"
[[ -z $should_be_running_release ]] && false
if [[ -n $balena_device_uuid ]]; then
while ! [[ $(balena device "${balena_device_uuid}" | grep -E ^COMMIT | awk '{print $2}') =~ ${should_be_running_release} ]]; do
running_release_id="$(balena device "${balena_device_uuid}" | grep -E ^COMMIT | awk '{print $2}')"
printf 'please wait, device %s should be running %s, but is still running %s...\n' \
"${balena_device_uuid}" \
"${1}" \
"${running_release_id}"
sleep "$(( (RANDOM % 5) + 5 ))s"
done
fi
}
function get_os_version() {
local BALENARC_BALENA_URL
BALENARC_BALENA_URL="$(echo "${BALENA_API_URL}" | sed 's#https://api\.##g')"
local os_version
os_version=${OS_VERSION:-$(with_backoff balena os versions "${DEVICE_TYPE}" | head -n 1)}
echo "${os_version}"
}
function upload_release_asset() {
if [[ "$RELEASE_ASSETS_TEST" =~ true ]]; then
local release_id
release_id=${1:-1}
release_asset="$(find / -type f -name '*.png' | head -n 1)"
curl ${curl_opts} "https://api.${DNS_TLD}/resin/release_asset" \
--header "Authorization: Bearer $(cat <~/.balena/token)" \
--form "asset=@${release_asset}" \
--form "release=${release_id}" \
--form "asset_key=$((RANDOM))-$(basename ${release_asset})" \
| jq -re .asset.href \
| xargs curl ${curl_opts} -o "/tmp/$((RANDOM))-$(basename ${release_asset})"
fi
}
# --- main
if [[ "$PRODUCTION_MODE" =~ true ]]; then
exit
fi
if [[ -n "${BALENA_DEVICE_UUID}" ]]; then
# prepend the device UUID if running on balenaOS
TLD="${BALENA_DEVICE_UUID}.${DNS_TLD}"
else
TLD="${DNS_TLD}"
fi
BALENA_API_URL=${BALENA_API_URL:-https://api.balena-cloud.com}
BALENARC_BALENA_URL="${DNS_TLD}"
CERTS=${CERTS:-/certs}
CONF=${CONF:-/balena/${TLD}.env}
DEVICE_TYPE=${DEVICE_TYPE:-generic-amd64}
GUEST_DISK_SIZE=${GUEST_DISK_SIZE:-8}
GUEST_IMAGE=${GUEST_IMAGE:-/balena/balena.img}
OS_VERSION="$(get_os_version)"
TEST_FLEET=${TEST_FLEET:-test-fleet}
[[ -f "$CONF" ]] && source "${CONF}"
update_ca_certificates # ensure self-signed root CA certificate(s) trust
registry_auth # optionally authenticate with DockerHub (rate-limiting)
wait_for_api # spin here until the API is responding
balena whoami || open_balena_login # spin here until authenticated
create_fleet # spin here until the fleet is created
# critical section
set_update_lock
download_os_image
configure_virtual_device
deploy_release
upload_release_asset "$(get_release_id)" # upload an additional asset to a release
remove_update_lock
# .. end
wait_for_device # spin here until test-device comes online
check_running_release # .. and ensure the device is running our release

32
src/balena-tests/functions Normal file
View File

@ -0,0 +1,32 @@
# https://coderwall.com/p/--eiqg/exponential-backoff-in-bash
function with_backoff() {
local max_attempts=${ATTEMPTS-5}
local timeout=${TIMEOUT-1}
local attempt=0
local exitCode=0
set +e
while [[ $attempt < $max_attempts ]]
do
"$@"
exitCode=$?
if [[ $exitCode == 0 ]]
then
break
fi
echo "Failure! Retrying in $timeout.." 1>&2
sleep "$timeout"
attempt=$(( attempt + 1 ))
timeout=$(( timeout * 2 ))
done
if [[ $exitCode != 0 ]]
then
echo "You've failed me for the last time! ($*)" 1>&2
fi
set -e
return $exitCode
}

4
src/cert-manager/Dockerfile Normal file
View File

@ -0,0 +1,4 @@
# https://github.com/balena-io/cert-manager
FROM balena/cert-manager:v0.2.2
COPY *.json /opt/

63
src/cert-manager/certs.json Normal file
View File

@ -0,0 +1,63 @@
[
{
"request": {
"key": {
"algo": "${key_algo}",
"size": ${key_size}
},
"hosts": ${hosts},
"names": [
{
"C": "${country}",
"L": "${locality_name}",
"O": "${org}",
"OU": "${org_unit}",
"ST": "${state}"
}
],
"CN": "${TLD}"
}
},
{
"request": {
"key": {
"algo": "${key_algo}",
"size": ${key_size}
},
"hosts": [
"vpn.${TLD}"
],
"names": [
{
"C": "${country}",
"L": "${locality_name}",
"O": "${org}",
"OU": "${org_unit}",
"ST": "${state}"
}
],
"CN": "vpn.${TLD}"
}
},
{
"request": {
"key": {
"algo": "${key_algo}",
"size": ${key_size}
},
"hosts": [
"api.${TLD}"
],
"names": [
{
"C": "${country}",
"L": "${locality_name}",
"O": "${org}",
"OU": "${org_unit}",
"ST": "${state}"
}
],
"CN": "api.${TLD}"
}
}
]

1
src/cert-manager/keys.json Normal file
View File

@ -0,0 +1 @@
[]

22
src/cert-provider/Dockerfile
View File

@ -1,22 +0,0 @@
FROM alpine
EXPOSE 80
WORKDIR /usr/src/app
VOLUME [ "/usr/src/app/certs" ]
RUN apk add --update bash curl git openssl ncurses socat
# from https://github.com/Neilpang/acme.sh/releases/tag/3.0.1
RUN git clone https://github.com/Neilpang/acme.sh.git && \
cd acme.sh && \
git fetch && git fetch --tags && \
git checkout 3.0.1 . && \
./acme.sh --install \
--cert-home /usr/src/app/certs
COPY entry.sh /entry.sh
COPY cert-provider.sh ./cert-provider.sh
COPY fake-le-bundle.pem ./
ENTRYPOINT [ "/entry.sh" ]
CMD [ "/usr/src/app/cert-provider.sh" ]

190
src/cert-provider/cert-provider.sh
View File

@ -1,190 +0,0 @@
#!/usr/bin/env bash
# the acme.sh client script, installed via Git in the Dockerfile...
ACME_BIN="$(realpath ~/.acme.sh/acme.sh)"
# the path to a bundle of certs to verify a LetsEncrypt staging certificate until Apr 2036...
ACME_STAGING_CA="/usr/src/app/fake-le-bundle.pem"
# the path to a file which stores the last successful mode of certificate we acquired...
ACME_MODE_FILE="/usr/src/app/certs/last_run_mode"
# colour output helpers...
reset=$(tput -T xterm sgr0)
red=$(tput -T xterm setaf 1)
green=$(tput -T xterm setaf 2)
yellow=$(tput -T xterm setaf 3)
blue=$(tput -T xterm setaf 4)
logError() {
echo "${red}[Error]${reset} $1"
}
logWarn() {
echo "${yellow}[Warn]${reset} $1"
}
logInfo() {
echo "${blue}[Info]${reset} $1"
}
logSuccess() {
echo "${green}[Success]${reset} $1"
}
logErrorAndStop() {
logError "$1 [Stopping]"
while true; do
# do nothing forever...
sleep 60
done
}
retryWithDelay() {
RETRIES=${2:-3}
DELAY=${3:-5}
local ATTEMPT=0
while [ "$RETRIES" -gt "$ATTEMPT" ]; do
(( ATTEMPT++ ))
logInfo "($ATTEMPT/$RETRIES) Connecting..."
if $1; then
logInfo "($ATTEMPT/$RETRIES) Success!"
return $?
fi
if [ "$RETRIES" -gt "$ATTEMPT" ]; then
logInfo "($ATTEMPT/$RETRIES) Failed. Retrying in ${DELAY} seconds..."
sleep "$DELAY"
else
logInfo "($ATTEMPT/$RETRIES) Failed!"
fi
done
return 1
}
waitForOnline() {
ADDRESS="${1,,}"
logInfo "Waiting for ${ADDRESS} to be available via HTTP..."
retryWithDelay "curl --output /dev/null --silent --head --fail --max-time 5 http://${ADDRESS}"
}
isUsingStagingCert() {
HOST="${1,,}"
echo "" | openssl s_client -host "$HOST" -port 443 -showcerts 2>/dev/null | awk '/BEGIN CERT/ {p=1} ; p==1; /END CERT/ {p=0}' | openssl verify -CAfile "$ACME_STAGING_CA" > /dev/null 2>&1
}
pre-flight() {
case "$ACTIVE" in
"true"|"yes")
;;
*)
logError "ACTIVE variable is not enabled. Value should be \"true\" or \"yes\" to continue."
return 1
;;
esac
if [ -z "$DOMAINS" ]; then
logError "DOMAINS must be set. Value should be a comma-delimited string of domains."
return 1
else
IFS=, read -r -a ACME_DOMAINS <<< "$DOMAINS"
IFS=' ' read -r -a ACME_DOMAIN_ARGS <<< "${ACME_DOMAINS[@]/#/-d }"
fi
if [ -z "$VALIDATION" ]; then
logInfo "VALIDATION not set. Using default: http-01"
VALIDATION="http-01"
else
case "$VALIDATION" in
"http-01")
logInfo "Using validation method: $VALIDATION"
;;
*)
logError "VALIDATION is invalid. Use a valid value: http-01"
return 1
;;
esac
fi
if [ -z "$OUTPUT_PEM" ]; then
logError "OUTPUT_PEM must be set. Value should be the path to install your certificate to."
return 1
fi
}
waitToSeeStagingCert() {
logInfo "Waiting for ${ACME_DOMAINS[0]} to use a staging certificate..."
retryWithDelay "isUsingStagingCert ${ACME_DOMAINS[0]}" 3 5
}
lastAcquiredCertFor() {
ACME_MODE="${1:-none}"
ACME_LAST_MODE="$(cat $ACME_MODE_FILE || echo '')"
logInfo "Last acquired certificate for ${ACME_LAST_MODE^^}"
[ "${ACME_LAST_MODE,,}" == "${ACME_MODE,,}" ]
}
acquireCertificate() {
ACME_MODE="${1:-staging}"
ACME_FORCE="${2:-false}"
ACME_OPTS=()
if [ "${ACME_FORCE,,}" == "true" ];then ACME_OPTS+=("--force"); fi
case "$ACME_MODE" in
"production")
logInfo "Using PRODUCTION mode"
;;
*)
logInfo "Using STAGING mode"
ACME_OPTS+=("--staging")
;;
esac
case "$VALIDATION" in
"http-01")
ACME_OPTS+=("--standalone")
;;
*)
logError "VALIDATION is invalid. Use a valid value: http-01"
return 1
;;
esac
if ! waitForOnline "${ACME_DOMAINS[0]}"; then
logError "Unable to access site over HTTP"
return 1
fi
logInfo "Issuing certificates..."
"$ACME_BIN" --server letsencrypt --issue "${ACME_OPTS[@]}" "${ACME_DOMAIN_ARGS[@]}"
logInfo "Installing certificates..." && \
"$ACME_BIN" --install-cert "${ACME_DOMAIN_ARGS[@]}" \
--cert-file /tmp/cert.pem \
--key-file /tmp/key.pem \
--fullchain-file /tmp/fullchain.pem \
--reloadcmd "cat /tmp/fullchain.pem /tmp/key.pem > $OUTPUT_PEM" && \
echo "${ACME_MODE}" > "${ACME_MODE_FILE}"
}
pre-flight || logErrorAndStop "Unable to continue due to misconfiguration. See errors above."
while ! waitForOnline "${ACME_DOMAINS[0]}"; do
logInfo "Unable to access ${ACME_DOMAINS[0]} on port 80. This is needed for certificate validation. Retrying in 30 seconds..."
sleep 30
done
if ! lastAcquiredCertFor "production"; then
acquireCertificate "staging" || logErrorAndStop "Unable to acquire a staging certificate."
waitToSeeStagingCert || logErrorAndStop "Unable to detect certificate change over. Cannot issue a production certificate."
acquireCertificate "production" "true" || logErrorAndStop "Unable to acquire a production certificate."
fi
logSuccess "Done!"
logInfo "Running cron..."
crond -f -d 7

3
src/cert-provider/entry.sh
View File

@ -1,3 +0,0 @@
#!/bin/bash
exec "$@"

119
src/cert-provider/fake-le-bundle.pem
View File

@ -1,119 +0,0 @@
-----BEGIN CERTIFICATE-----
MIIDrzCCApegAwIBAgIRALqMZiRNaRF4EGZS9urlj+0wDQYJKoZIhvcNAQELBQAw
cTELMAkGA1UEBhMCVVMxMzAxBgNVBAoTKihTVEFHSU5HKSBJbnRlcm5ldCBTZWN1
cml0eSBSZXNlYXJjaCBHcm91cDEtMCsGA1UEAxMkKFNUQUdJTkcpIERvY3RvcmVk
IER1cmlhbiBSb290IENBIFgzMB4XDTAwMDkzMDIxMTIxOVoXDTIxMDEzMDE0MDEx
NVowcTELMAkGA1UEBhMCVVMxMzAxBgNVBAoTKihTVEFHSU5HKSBJbnRlcm5ldCBT
ZWN1cml0eSBSZXNlYXJjaCBHcm91cDEtMCsGA1UEAxMkKFNUQUdJTkcpIERvY3Rv
cmVkIER1cmlhbiBSb290IENBIFgzMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB
CgKCAQEAqUZjoRbjgXecPWxXkGCUEXcNrupL7dkbwc0jUTLFEDvcyfD1gYekY5uL
D19uzYTl0pKZzzDXHJPnJY5EEp27nACFOm8XzX9sORAangP0OnGUkXJZDHM+8cX2
EHJbfj0lg1JirRF3w2u1/KRuFEvIlWg3FdXdsSFHBF5z1Ij7MLn7Ska5c/5fKsDW
EYzOMB6EBW1T9RDkVk/Q965EwDT4bR6BOXakasgfKrH9m1f6l9MmA0VnXdw9rZ+s
TvMHG1yWBqNMSqCKe3jG6caWgN7llEbj5YsCWs32bz2dMftGkXBPcy1fNWvpeT7G
Dz2Z0QWTlHkyXA2kGw32fdoXLHWOEwIDAQABo0IwQDAOBgNVHQ8BAf8EBAMCAQYw
DwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUCFfaiceiU3kMT93gkI90uuInc0Qw
DQYJKoZIhvcNAQELBQADggEBAF7lEtHuSN4j+xFQsM/ujaVKcn57VbrbTecnspmJ
JA7Hrn6OErshGNO0p1/u14c7tGHKjtF1tEFFSVhbNXlKw9O99AfhmlFgdGcJKEHn
ZctBB8bhNO387vbiCYIHdU/nSba9MCDYw2/UCtobZ6ao+KJA3IKmPixctAbn2Ikr
EN9X0SXNP1gnqQP4VhZJIh6cd7rg9MimzoLlMI3m2z11dSGYbh8OWSdvA7aLbSGo
gDO5H4WD8fgqEG0reSBO89eeH+we+BZxQtBiU3b9VMV0drc+7zC2NbXqeQwu6QTl
fbJ8ytqcqUy0g5XSE6WCzPOL3H9r0j9G64dfotGlBA5tG6w=
-----END CERTIFICATE-----
-----BEGIN CERTIFICATE-----
MIIFmDCCA4CgAwIBAgIQU9C87nMpOIFKYpfvOHFHFDANBgkqhkiG9w0BAQsFADBm
MQswCQYDVQQGEwJVUzEzMDEGA1UEChMqKFNUQUdJTkcpIEludGVybmV0IFNlY3Vy
aXR5IFJlc2VhcmNoIEdyb3VwMSIwIAYDVQQDExkoU1RBR0lORykgUHJldGVuZCBQ
ZWFyIFgxMB4XDTE1MDYwNDExMDQzOFoXDTM1MDYwNDExMDQzOFowZjELMAkGA1UE
BhMCVVMxMzAxBgNVBAoTKihTVEFHSU5HKSBJbnRlcm5ldCBTZWN1cml0eSBSZXNl
YXJjaCBHcm91cDEiMCAGA1UEAxMZKFNUQUdJTkcpIFByZXRlbmQgUGVhciBYMTCC
AiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBALbagEdDTa1QgGBWSYkyMhsc
ZXENOBaVRTMX1hceJENgsL0Ma49D3MilI4KS38mtkmdF6cPWnL++fgehT0FbRHZg
jOEr8UAN4jH6omjrbTD++VZneTsMVaGamQmDdFl5g1gYaigkkmx8OiCO68a4QXg4
wSyn6iDipKP8utsE+x1E28SA75HOYqpdrk4HGxuULvlr03wZGTIf/oRt2/c+dYmD
oaJhge+GOrLAEQByO7+8+vzOwpNAPEx6LW+crEEZ7eBXih6VP19sTGy3yfqK5tPt
TdXXCOQMKAp+gCj/VByhmIr+0iNDC540gtvV303WpcbwnkkLYC0Ft2cYUyHtkstO
fRcRO+K2cZozoSwVPyB8/J9RpcRK3jgnX9lujfwA/pAbP0J2UPQFxmWFRQnFjaq6
rkqbNEBgLy+kFL1NEsRbvFbKrRi5bYy2lNms2NJPZvdNQbT/2dBZKmJqxHkxCuOQ
FjhJQNeO+Njm1Z1iATS/3rts2yZlqXKsxQUzN6vNbD8KnXRMEeOXUYvbV4lqfCf8
mS14WEbSiMy87GB5S9ucSV1XUrlTG5UGcMSZOBcEUpisRPEmQWUOTWIoDQ5FOia/
GI+Ki523r2ruEmbmG37EBSBXdxIdndqrjy+QVAmCebyDx9eVEGOIpn26bW5LKeru
mJxa/CFBaKi4bRvmdJRLAgMBAAGjQjBAMA4GA1UdDwEB/wQEAwIBBjAPBgNVHRMB
Af8EBTADAQH/MB0GA1UdDgQWBBS182Xy/rAKkh/7PH3zRKCsYyXDFDANBgkqhkiG
9w0BAQsFAAOCAgEAncDZNytDbrrVe68UT6py1lfF2h6Tm2p8ro42i87WWyP2LK8Y
nLHC0hvNfWeWmjZQYBQfGC5c7aQRezak+tHLdmrNKHkn5kn+9E9LCjCaEsyIIn2j
qdHlAkepu/C3KnNtVx5tW07e5bvIjJScwkCDbP3akWQixPpRFAsnP+ULx7k0aO1x
qAeaAhQ2rgo1F58hcflgqKTXnpPM02intVfiVVkX5GXpJjK5EoQtLceyGOrkxlM/
sTPq4UrnypmsqSagWV3HcUlYtDinc+nukFk6eR4XkzXBbwKajl0YjztfrCIHOn5Q
CJL6TERVDbM/aAPly8kJ1sWGLuvvWYzMYgLzDul//rUF10gEMWaXVZV51KpS9DY/
5CunuvCXmEQJHo7kGcViT7sETn6Jz9KOhvYcXkJ7po6d93A/jy4GKPIPnsKKNEmR
xUuXY4xRdh45tMJnLTUDdC9FIU0flTeO9/vNpVA8OPU1i14vCz+MU8KX1bV3GXm/
fxlB7VBBjX9v5oUep0o/j68R/iDlCOM4VVfRa8gX6T2FU7fNdatvGro7uQzIvWof
gN9WUwCbEMBy/YhBSrXycKA8crgGg3x1mIsopn88JKwmMBa68oS7EHM9w7C4y71M
7DiA+/9Qdp9RBWJpTS9i/mDnJg1xvo8Xz49mrrgfmcAXTCJqXi24NatI3Oc=
-----END CERTIFICATE-----
-----BEGIN CERTIFICATE-----
MIICTjCCAdSgAwIBAgIRAIPgc3k5LlLVLtUUvs4K/QcwCgYIKoZIzj0EAwMwaDEL
MAkGA1UEBhMCVVMxMzAxBgNVBAoTKihTVEFHSU5HKSBJbnRlcm5ldCBTZWN1cml0
eSBSZXNlYXJjaCBHcm91cDEkMCIGA1UEAxMbKFNUQUdJTkcpIEJvZ3VzIEJyb2Nj
b2xpIFgyMB4XDTIwMDkwNDAwMDAwMFoXDTQwMDkxNzE2MDAwMFowaDELMAkGA1UE
BhMCVVMxMzAxBgNVBAoTKihTVEFHSU5HKSBJbnRlcm5ldCBTZWN1cml0eSBSZXNl
YXJjaCBHcm91cDEkMCIGA1UEAxMbKFNUQUdJTkcpIEJvZ3VzIEJyb2Njb2xpIFgy
MHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEOvS+w1kCzAxYOJbA06Aw0HFP2tLBLKPo
FQqR9AMskl1nC2975eQqycR+ACvYelA8rfwFXObMHYXJ23XLB+dAjPJVOJ2OcsjT
VqO4dcDWu+rQ2VILdnJRYypnV1MMThVxo0IwQDAOBgNVHQ8BAf8EBAMCAQYwDwYD
VR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQU3tGjWWQOwZo2o0busBB2766XlWYwCgYI
KoZIzj0EAwMDaAAwZQIwRcp4ZKBsq9XkUuN8wfX+GEbY1N5nmCRc8e80kUkuAefo
uc2j3cICeXo1cOybQ1iWAjEA3Ooawl8eQyR4wrjCofUE8h44p0j7Yl/kBlJZT8+9
vbtH7QiVzeKCOTQPINyRql6P
-----END CERTIFICATE-----
-----BEGIN CERTIFICATE-----
MIIFWzCCA0OgAwIBAgIQTfQrldHumzpMLrM7jRBd1jANBgkqhkiG9w0BAQsFADBm
MQswCQYDVQQGEwJVUzEzMDEGA1UEChMqKFNUQUdJTkcpIEludGVybmV0IFNlY3Vy
aXR5IFJlc2VhcmNoIEdyb3VwMSIwIAYDVQQDExkoU1RBR0lORykgUHJldGVuZCBQ
ZWFyIFgxMB4XDTIwMDkwNDAwMDAwMFoXDTI1MDkxNTE2MDAwMFowWTELMAkGA1UE
BhMCVVMxIDAeBgNVBAoTFyhTVEFHSU5HKSBMZXQncyBFbmNyeXB0MSgwJgYDVQQD
Ex8oU1RBR0lORykgQXJ0aWZpY2lhbCBBcHJpY290IFIzMIIBIjANBgkqhkiG9w0B
AQEFAAOCAQ8AMIIBCgKCAQEAu6TR8+74b46mOE1FUwBrvxzEYLck3iasmKrcQkb+
gy/z9Jy7QNIAl0B9pVKp4YU76JwxF5DOZZhi7vK7SbCkK6FbHlyU5BiDYIxbbfvO
L/jVGqdsSjNaJQTg3C3XrJja/HA4WCFEMVoT2wDZm8ABC1N+IQe7Q6FEqc8NwmTS
nmmRQm4TQvr06DP+zgFK/MNubxWWDSbSKKTH5im5j2fZfg+j/tM1bGaczFWw8/lS
nukyn5J2L+NJYnclzkXoh9nMFnyPmVbfyDPOc4Y25aTzVoeBKXa/cZ5MM+WddjdL
biWvm19f1sYn1aRaAIrkppv7kkn83vcth8XCG39qC2ZvaQIDAQABo4IBEDCCAQww
DgYDVR0PAQH/BAQDAgGGMB0GA1UdJQQWMBQGCCsGAQUFBwMCBggrBgEFBQcDATAS
BgNVHRMBAf8ECDAGAQH/AgEAMB0GA1UdDgQWBBTecnpI3zHDplDfn4Uj31c3S10u
ZTAfBgNVHSMEGDAWgBS182Xy/rAKkh/7PH3zRKCsYyXDFDA2BggrBgEFBQcBAQQq
MCgwJgYIKwYBBQUHMAKGGmh0dHA6Ly9zdGcteDEuaS5sZW5jci5vcmcvMCsGA1Ud
HwQkMCIwIKAeoByGGmh0dHA6Ly9zdGcteDEuYy5sZW5jci5vcmcvMCIGA1UdIAQb
MBkwCAYGZ4EMAQIBMA0GCysGAQQBgt8TAQEBMA0GCSqGSIb3DQEBCwUAA4ICAQCN
DLam9yN0EFxxn/3p+ruWO6n/9goCAM5PT6cC6fkjMs4uas6UGXJjr5j7PoTQf3C1
vuxiIGRJC6qxV7yc6U0X+w0Mj85sHI5DnQVWN5+D1er7mp13JJA0xbAbHa3Rlczn
y2Q82XKui8WHuWra0gb2KLpfboYj1Ghgkhr3gau83pC/WQ8HfkwcvSwhIYqTqxoZ
Uq8HIf3M82qS9aKOZE0CEmSyR1zZqQxJUT7emOUapkUN9poJ9zGc+FgRZvdro0XB
yphWXDaqMYph0DxW/10ig5j4xmmNDjCRmqIKsKoWA52wBTKKXK1na2ty/lW5dhtA
xkz5rVZFd4sgS4J0O+zm6d5GRkWsNJ4knotGXl8vtS3X40KXeb3A5+/3p0qaD215
Xq8oSNORfB2oI1kQuyEAJ5xvPTdfwRlyRG3lFYodrRg6poUBD/8fNTXMtzydpRgy
zUQZh/18F6B/iW6cbiRN9r2Hkh05Om+q0/6w0DdZe+8YrNpfhSObr/1eVZbKGMIY
qKmyZbBNu5ysENIK5MPc14mUeKmFjpN840VR5zunoU52lqpLDua/qIM8idk86xGW
xx2ml43DO/Ya/tVZVok0mO0TUjzJIfPqyvr455IsIut4RlCR9Iq0EDTve2/ZwCuG
hSjpTUFGSiQrR2JK2Evp+o6AETUkBCO1aw0PpQBPDQ==
-----END CERTIFICATE-----
-----BEGIN CERTIFICATE-----
MIIDCzCCApGgAwIBAgIRALRY4992FVxZJKOJ3bpffWIwCgYIKoZIzj0EAwMwaDEL
MAkGA1UEBhMCVVMxMzAxBgNVBAoTKihTVEFHSU5HKSBJbnRlcm5ldCBTZWN1cml0
eSBSZXNlYXJjaCBHcm91cDEkMCIGA1UEAxMbKFNUQUdJTkcpIEJvZ3VzIEJyb2Nj
b2xpIFgyMB4XDTIwMDkwNDAwMDAwMFoXDTI1MDkxNTE2MDAwMFowVTELMAkGA1UE
BhMCVVMxIDAeBgNVBAoTFyhTVEFHSU5HKSBMZXQncyBFbmNyeXB0MSQwIgYDVQQD
ExsoU1RBR0lORykgRXJzYXR6IEVkYW1hbWUgRTEwdjAQBgcqhkjOPQIBBgUrgQQA
IgNiAAT9v/PJUtHOTk28nXCXrpP665vI4Z094h8o7R+5E6yNajZa0UubqjpZFoGq
u785/vGXj6mdfIzc9boITGusZCSWeMj5ySMZGZkS+VSvf8VQqj+3YdEu4PLZEjBA
ivRFpEejggEQMIIBDDAOBgNVHQ8BAf8EBAMCAYYwHQYDVR0lBBYwFAYIKwYBBQUH
AwIGCCsGAQUFBwMBMBIGA1UdEwEB/wQIMAYBAf8CAQAwHQYDVR0OBBYEFOv5JcKA
KGbibQiSMvPC4a3D/zVFMB8GA1UdIwQYMBaAFN7Ro1lkDsGaNqNG7rAQdu+ul5Vm
MDYGCCsGAQUFBwEBBCowKDAmBggrBgEFBQcwAoYaaHR0cDovL3N0Zy14Mi5pLmxl
bmNyLm9yZy8wKwYDVR0fBCQwIjAgoB6gHIYaaHR0cDovL3N0Zy14Mi5jLmxlbmNy
Lm9yZy8wIgYDVR0gBBswGTAIBgZngQwBAgEwDQYLKwYBBAGC3xMBAQEwCgYIKoZI
zj0EAwMDaAAwZQIwXcZbdgxcGH9rTErfSTkXfBKKygU0yO7OpbuNeY1id0FZ/hRY
N5fdLOGuc+aHfCsMAjEA0P/xwKr6NQ9MN7vrfGAzO397PApdqfM7VdFK18aEu1xm
3HMFKzIR8eEPsMx4smMl
-----END CERTIFICATE-----

7
src/haproxy-sidecar/Dockerfile Normal file
View File

@ -0,0 +1,7 @@
FROM docker
COPY balena.sh /usr/local/bin/balena.sh
RUN apk add --no-cache bash
CMD /usr/local/bin/balena.sh

75
src/haproxy-sidecar/balena.sh Executable file
View File

@ -0,0 +1,75 @@
#!/usr/bin/env bash
set -ea
[[ $VERBOSE =~ on|On|Yes|yes|true|True ]] && set -x
function cleanup() {
rm -f /host/run/docker.sock
# crash loop backoff
sleep "$(( (RANDOM % 5) + 5 ))s"
}
trap 'cleanup' EXIT
if [ -S /host/run/balena-engine.sock ]; then
ln -s /host/run/balena-engine.sock /host/run/docker.sock
fi
which curl || apk add curl --no-cache
which jq || apk add jq --no-cache
if docker inspect "${BALENA_APP_UUID}_default" --format "{{.ID}}"; then
network="${BALENA_APP_UUID}_default"
elif docker inspect "${BALENA_APP_ID}_default" --format "{{.ID}}"; then
network="${BALENA_APP_ID}_default"
else
network=open-balena_default
fi
# shellcheck disable=SC2153
for alias in ${ALIASES//,/ }; do
hostname="${alias}.${DNS_TLD}"
aliases="--alias ${hostname} ${aliases}"
done
while true; do
if [[ -n $BALENA_SUPERVISOR_ADDRESS ]] && [[ -n $BALENA_SUPERVISOR_API_KEY ]]; then
while [[ "$(curl --silent --retry 3 --fail \
"${BALENA_SUPERVISOR_ADDRESS}/v1/device?apikey=${BALENA_SUPERVISOR_API_KEY}" \
-H "Content-Type:application/json" | jq -r '.update_pending')" =~ true ]]; do
sleep "$(( (RANDOM % 3) + 3 ))s"
done
sleep "$(( (RANDOM % 5) + 5 ))s"
fi
while [[ "$(docker ps \
--filter "name=haproxy" \
--filter "expose=1936/tcp" \
--filter "status=running" \
--filter "network=${network}" \
--format "{{.ID}}")" == '' ]]; do
sleep "$(( (RANDOM % 3) + 3 ))s"
done
haproxy="$(docker ps \
--filter "name=haproxy" \
--filter "expose=1936/tcp" \
--filter "status=running" \
--filter "network=${network}" \
--format "{{.ID}}")"
if ! [[ $restarted == "${haproxy}" ]]; then
docker network disconnect "${network}" "${haproxy}"
# shellcheck disable=SC2086
docker network connect --alias haproxy ${aliases} "${network}" "${haproxy}"
docker restart "${haproxy}"
restarted="${haproxy}"
fi
sleep "$(( (RANDOM % 15) + 15 ))s"
done

10
src/haproxy/Dockerfile
View File

@ -1,10 +1,4 @@
FROM haproxy:2.9-alpine
VOLUME [ "/certs" ]
RUN apk add --update inotify-tools
# https://github.com/balena-io/open-balena-haproxy
FROM balena/open-balena-haproxy:v4.3.1
COPY haproxy.cfg /usr/local/etc/haproxy/haproxy.cfg
COPY start-haproxy.sh /start-haproxy
CMD /start-haproxy

252
src/haproxy/haproxy.cfg
View File

@ -1,134 +1,176 @@
global
tune.ssl.default-dh-param 1024
# https://github.com/haproxytech/haproxy-lua-cors
lua-load /usr/local/etc/haproxy/cors.lua
# https://www.haproxy.com/blog/introduction-to-haproxy-logging/
log stdout format raw daemon "${LOGLEVEL}"
log stderr format raw daemon "${LOGLEVEL}"
ssl-default-bind-options ssl-min-ver TLSv1.2
defaults
timeout connect 5s
timeout client 50s
timeout server 50s
frontend http-in
balance roundrobin
default-server init-addr last,libc,none
default-server inter 3s rise 2 fall 3
log global
mode http
option contstats
option dontlognull
option forwardfor
bind *:80
reqadd X-Forwarded-Proto:\ http
option httplog
timeout client 63s
timeout connect 5s
timeout http-keep-alive 1s
timeout http-request 63s
timeout server 63s
timeout tunnel 3600s
acl is_cert_validation path -i -m beg "/.well-known/acme-challenge/"
use_backend cert-provider if is_cert_validation
resolvers docker-bridge-resolver
nameserver docker-resolver 127.0.0.11:53
hold valid 0ms
acl host_api hdr_dom(host) -i "api.${HAPROXY_HOSTNAME}"
use_backend backend_api if host_api
http-errors balena-http-errors
errorfile 400 /etc/haproxy/errors/400.http
errorfile 401 /etc/haproxy/errors/401.http
errorfile 403 /etc/haproxy/errors/403.http
errorfile 404 /etc/haproxy/errors/404.http
errorfile 500 /etc/haproxy/errors/500.http
errorfile 502 /etc/haproxy/errors/502.http
errorfile 503 /etc/haproxy/errors/503.http
acl host_registry hdr_dom(host) -i "registry.${HAPROXY_HOSTNAME}"
use_backend backend_registry if host_registry
userlist balena
user balena insecure-password "${BALENA_DEVICE_UUID}"
acl host_vpn hdr_dom(host) -i "vpn.${HAPROXY_HOSTNAME}"
use_backend backend_vpn if host_vpn
listen haproxy-stats
bind :::1936 v4v6 ssl crt "${CERT_CHAIN_PATH}" alpn h2,http/1.1
stats auth "balena:${BALENA_DEVICE_UUID}"
stats enable
stats uri /metrics
acl host_s3 hdr_dom(host) -i "s3.${HAPROXY_HOSTNAME}"
use_backend backend_s3 if host_s3
frontend http
bind :::80 v4v6
default_backend api-backend
errorfiles balena-http-errors
http-request capture req.hdr(Host) len 15
http-response lua.cors
# https://www.haproxy.com/blog/haproxy-log-customization/
log-format "%ci:%cp [%tr] %ft %b/%s %TR/%Tw/%Tc/%Tr/%Ta %ST %B %CC %CS %tsc %ac/%fc/%bc/%sc/%rc %sq/%bq %hr %hs %{+Q}r"
frontend ssl-in
acl api_dead nbsrv(api-backend) lt 1
acl registry_dead nbsrv(registry-backend) lt 1
acl vpn_dead nbsrv(vpn-backend) lt 1
monitor-uri /health
monitor fail if api_dead registry_dead vpn_dead
acl host-api-backend hdr_beg(host) -i "api."
# default public device URL(s) always go to the API
acl host-pdu-default hdr(host) -m reg -i "\.?([0-9a-f]{32}|${BALENA_DEVICE_UUID})\.(devices|balena-?(.*)-devices)\."
use_backend api-backend if host-api-backend || host-pdu-default
acl host-registry-backend hdr_beg(host) -i "registry2."
http-request add-header X-Forwarded-Proto http if host-registry-backend
use_backend registry-backend if host-registry-backend
acl host-s3-backend hdr_beg(host) -i "s3."
http-request add-header X-Forwarded-Proto http if host-s3-backend
use_backend s3-backend if host-s3-backend
acl host-minio-backend hdr_beg(host) -i "minio."
http-request add-header X-Forwarded-Proto http if host-minio-backend
use_backend minio-backend if host-minio-backend
# routes between OpenVPN, SSL and HTTPS traffic
frontend tcp-router
mode tcp
bind *:443
option tcplog
log-format "%ci:%cp [%t] %ft %b/%s %Tw/%Tc/%Tt %B %ts %ac/%fc/%bc/%sc/%rc %sq/%bq"
bind :::443 v4v6
tcp-request inspect-delay 2s
tcp-request content accept if { req.ssl_hello_type 1 }
acl is_ssl req.ssl_ver 2:3.4
acl host_tunnel req_ssl_sni -i "tunnel.${HAPROXY_HOSTNAME}"
use_backend redirect-to-tunnel-in if host_tunnel
acl sni-host-tunnel req_ssl_sni -m beg "tunnel."
use_backend redirect-to-tunnel if sni-host-tunnel
use_backend redirect-to-https-in if is_ssl
use_backend vpn-devices if !is_ssl
# everything else => HTTPS
use_backend redirect-to-https if is_ssl
backend redirect-to-https-in
# or VPN
use_backend vpn-backend if !is_ssl
backend redirect-to-tunnel
mode tcp
server localhost 127.0.0.1:3129 send-proxy-v2
# https://stackoverflow.com/a/39213442/1559300
listen tunnel-backend
mode tcp
option tcplog
log-format "%ci:%cp [%t] %ft %b/%s %Tw/%Tc/%Tt %B %ts %ac/%fc/%bc/%sc/%rc %sq/%bq"
bind 127.0.0.1:3129 ssl crt "${CERT_CHAIN_PATH}" alpn h2,http/1.1 accept-proxy
server tunnel vpn:3128 resolvers docker-bridge-resolver resolve-prefer ipv4 check port 3128
backend redirect-to-https
mode tcp
balance roundrobin
server localhost 127.0.0.1:444 send-proxy-v2
backend redirect-to-tunnel-in
frontend https
bind 127.0.0.1:444 ssl crt "${CERT_CHAIN_PATH}" alpn h2,http/1.1 accept-proxy
default_backend api-backend
errorfiles balena-http-errors
http-request add-header X-Forwarded-Proto https
http-request capture req.hdr(Host) len 15
http-response lua.cors
log-format "%ci:%cp [%tr] %ft %b/%s %TR/%Tw/%Tc/%Tr/%Ta %ST %B %CC %CS %tsc %ac/%fc/%bc/%sc/%rc %sq/%bq %hr %hs %{+Q}r"
acl host-api-backend hdr_beg(host) -i "api."
use_backend api-backend if host-api-backend
acl host-registry-backend hdr_beg(host) -i "registry2."
use_backend registry-backend if host-registry-backend
acl host-s3-backend hdr_beg(host) -i "s3."
use_backend s3-backend if host-s3-backend
acl host-minio-backend hdr_beg(host) -i "minio."
use_backend minio-backend if host-minio-backend
acl host-ca-backend hdr_beg(host) -i "ca."
# only allow CRL requests unauthenticated, protect everything else
acl balena-ca-crl path -i -m beg /api/v1/cfssl/crl
acl balena-ca-auth http_auth(balena)
http-request auth realm balena-ca if host-ca-backend !balena-ca-auth !balena-ca-crl
use_backend ca-backend if host-ca-backend
acl host-ocsp-backend hdr_beg(host) -i "ocsp."
use_backend ocsp-backend if host-ocsp-backend
backend api-backend
server api api:80 resolvers docker-bridge-resolver resolve-prefer ipv4 check port 80
backend registry-backend
server registry registry:80 resolvers docker-bridge-resolver resolve-prefer ipv4 check port 80
backend s3-backend
server s3 s3:80 resolvers docker-bridge-resolver resolve-prefer ipv4 check port 80
# https://github.com/minio/console
backend minio-backend
server s3-console s3:43697 resolvers docker-bridge-resolver resolve-prefer ipv4 check port 43697
backend db-backend
mode tcp
balance roundrobin
server localhost 127.0.0.1:3129
server db db:5432 resolvers docker-bridge-resolver resolve-prefer ipv4 check port 5432
frontend https-in
mode http
option forwardfor
bind 127.0.0.1:444 ssl crt /etc/ssl/private/open-balena.pem accept-proxy
reqadd X-Forwarded-Proto:\ https
acl host_api hdr_dom(host) -i "api.${HAPROXY_HOSTNAME}"
use_backend backend_api if host_api
acl host_registry hdr_dom(host) -i "registry.${HAPROXY_HOSTNAME}"
use_backend backend_registry if host_registry
acl host_vpn hdr_dom(host) -i "vpn.${HAPROXY_HOSTNAME}"
use_backend backend_vpn if host_vpn
acl host_s3 hdr_dom(host) -i "s3.${HAPROXY_HOSTNAME}"
use_backend backend_s3 if host_s3
backend backend_api
mode http
option forwardfor
balance roundrobin
server balena_api_1 api:80 check port 80
backend backend_registry
mode http
option forwardfor
balance roundrobin
server balena_registry_1 registry:80 check port 80
backend backend_vpn
mode http
option forwardfor
balance roundrobin
server balena_vpn_1 vpn:80 check port 80
backend backend_s3
mode http
option forwardfor
balance roundrobin
server balena_s3_1 s3:80 check port 80
backend cert-provider
mode http
option forwardfor
balance roundrobin
server balena_cert-provider_1 cert-provider:80 no-check
backend vpn-devices
backend redis-backend
mode tcp
server balena_vpn_1 vpn:443 send-proxy-v2 check-send-proxy port 443
server redis redis:6379 resolvers docker-bridge-resolver resolve-prefer ipv4 check port 6379
frontend db
mode tcp
bind *:5432
default_backend backend_db
timeout client 1h
backend ca-backend
server cfssl-ca balena-ca:8888 resolvers docker-bridge-resolver resolve-prefer ipv4 check port 8888
backend backend_db
mode tcp
server balena_db_1 db:5432 check port 5432
backend ocsp-backend
server cfssl-ocsp balena-ca:8889 resolvers docker-bridge-resolver resolve-prefer ipv4 check port 8889
frontend redis
backend vpn-backend
mode tcp
bind *:6379
default_backend backend_redis
timeout client 1h
backend backend_redis
mode tcp
server balena_redis_1 redis:6379 check port 6379
listen vpn-tunnel
mode tcp
bind *:3128
server balena_vpn vpn:3128 check port 3128
listen vpn-tunnel-tls
mode tcp
bind *:3129 ssl crt /etc/ssl/private/open-balena.pem
server balena_vpn vpn:3128 check port 3128
server openvpn vpn:443 resolvers docker-bridge-resolver resolve-prefer ipv4 send-proxy-v2 check-send-proxy check port 443

32
src/haproxy/start-haproxy.sh
View File

@ -1,32 +0,0 @@
#!/bin/sh
OPENBALENA_CERT=/etc/ssl/private/open-balena.pem
mkdir -p "$(dirname "${OPENBALENA_CERT}")"
if [ -f "/certs/open-balena.pem" ]; then
echo "Using certificate from cert-provider..."
cp /certs/open-balena.pem "${OPENBALENA_CERT}"
else
echo "Building certificate from environment variables..."
(
echo "${BALENA_HAPROXY_CRT}" | base64 -d
echo "${BALENA_HAPROXY_KEY}" | base64 -d
echo "${BALENA_ROOT_CA}" | base64 -d
) > "${OPENBALENA_CERT}"
fi
haproxy -f /usr/local/etc/haproxy/haproxy.cfg -W &
HAPROXY_PID=$!
while true; do
inotifywait -r -e create -e modify -e delete /certs
if [ -f "/certs/open-balena.pem" ]; then
echo "Updating certificate from cert-provider..."
cp /certs/open-balena.pem "${OPENBALENA_CERT}"
fi
echo "Certificate change detected. Reloading..."
kill -SIGUSR2 $HAPROXY_PID
sleep 1;
done

5
src/tag-sidecar/Dockerfile Normal file
View File

@ -0,0 +1,5 @@
FROM bash:alpine3.19
COPY balena.sh /usr/local/bin/balena.sh
CMD /usr/local/bin/balena.sh

44
src/tag-sidecar/balena.sh Executable file
View File

@ -0,0 +1,44 @@
#!/usr/bin/env bash
set -ea
[[ $VERBOSE =~ on|On|Yes|yes|true|True ]] && set -x
[[ $ENABLED == 'true' ]] || exit
curl_with_opts() {
curl --fail --silent --retry 3 --connect-timeout 3 --compressed "$@"
}
get_aws_meta() {
if [[ $1 =~ ^.*/$ ]]; then
for key in $(curl_with_opts "$1"); do
get_aws_meta "$1${key}"
done
else
echo "$(echo "$1" | cut -c41-);$(curl_with_opts "$1" | tr '\n' ',')"
fi
}
if [[ -n $BALENA_API_URL ]] && [[ -n $BALENA_DEVICE_UUID ]] && [[ -n $BALENA_API_KEY ]]; then
which curl || apk add curl --no-cache
which jq || apk add jq --no-cache
device_id="$(curl_with_opts \
"${BALENA_API_URL}/v6/device?\$filter=uuid%20eq%20'${BALENA_DEVICE_UUID}'" \
-H "Content-Type: application/json" \
-H "Authorization: Bearer ${BALENA_API_KEY}" | jq -r .d[].id)"
for key in $(curl_with_opts http://169.254.169.254/latest/meta-data \
| grep -Ev 'iam|metrics|identity-credentials|network|events'); do
for kv in $(get_aws_meta "http://169.254.169.254/latest/meta-data/${key}"); do
tag_key="$(echo "${kv}" | awk -F';' '{print $1}')"
value="$(echo "${kv}" | awk -F';' '{print $2}')"
curl_with_opts "${BALENA_API_URL}/v6/device_tag" \
-H "Content-Type: application/json" \
-H "Authorization: Bearer ${BALENA_API_KEY}" \
--data "{\"device\":\"${device_id}\",\"tag_key\":\"${tag_key}\",\"value\":\"${value}\"}"
done
done
fi

11
src/test-device/Dockerfile Normal file
View File

@ -0,0 +1,11 @@
# https://hub.docker.com/r/qemux/qemu-docker
# https://github.com/qemus/qemu-docker
FROM qemux/qemu-docker:4.24
RUN apt-get update && apt-get install -y --no-install-recommends \
minicom \
&& rm -rf /var/lib/apt/lists/*
COPY balena.sh /usr/sbin/
WORKDIR /balena

38
src/test-device/balena.sh Executable file
View File

@ -0,0 +1,38 @@
#!/usr/bin/env bash
set -ae
[[ $VERBOSE =~ on|On|Yes|yes|true|True ]] && set -x
function cleanup() {
rm -f "${tmpimg}"
# crash loop backoff
sleep "$(( (RANDOM % 5) + 5 ))s"
}
trap 'cleanup' EXIT
if [[ "$PRODUCTION_MODE" =~ true ]]; then
exit
fi
until test -f "${GUEST_IMAGE%.*}.ready"; do sleep "$(( (RANDOM % 5) + 5 ))s"; done
tmpimg="$(mktemp)"
cat <"${GUEST_IMAGE}" >"${tmpimg}"
exec /usr/bin/qemu-system-x86_64 \
-bios /usr/share/ovmf/OVMF.fd \
-chardev socket,id=serial0,path=/run/console.sock,server=on,wait=off \
-cpu max \
-device ahci,id=ahci \
-device ide-hd,drive=disk,bus=ahci.0 \
-device virtio-net-pci,netdev=n1 \
-drive file="${tmpimg}",media=disk,cache=none,format=raw,if=none,id=disk \
-m "${MEMORY}" \
-machine q35 \
-netdev "user,id=n1,dns=127.0.0.1,guestfwd=tcp:10.0.2.100:80-cmd:netcat haproxy 80,guestfwd=tcp:10.0.2.100:443-cmd:netcat haproxy 443" \
-nodefaults \
-nographic \
-serial chardev:serial0 \
-smp "${CPU}"