open-balena/docker-compose.yml

---
version: '2.4'

volumes:
  builder-certs-ca: {}
  builder-certs-client: {}
  builder-data: {}
  cert-manager-data: {}
  certs-data: {}
  db-data: {}
  pki-data: {}
  redis-data: {}
  resin-data: {}
  s3-data: {}

x-default-healthcheck: &default-healthcheck
  test: /usr/src/app/docker-hc
  interval: 45s
  timeout: 15s
  retries: 3

x-default-environment: &default-environment
  # FIXME: hardcoded https://github.com/balena-io/open-balena-db/blob/master/create-resin-db.sh#L4
  DB_NAME: resin
  # FIXME: hardcoded https://github.com/balena-io/open-balena-db/blob/master/Dockerfile#L3-L4
  DB_PASSWORD: docker
  DB_USER: docker
  LOG_LEVEL: DEBUG
  PRODUCTION_MODE: 'false'

x-default-healthcheck-trait: &with-default-healthcheck
  healthcheck:
    <<: *default-healthcheck

x-default-volumes-trait: &with-default-volumes
  volumes:
    - certs-data:/certs
    - resin-data:/balena

x-default-privileges-trait: &with-default-privileges
  cap_add:
    - SYS_ADMIN
    - SYS_RESOURCE
  security_opt:
    - apparmor=unconfined
  tmpfs:
    - /run
    - /sys/fs/cgroup

x-extended-privileges-trait: &with-extended-privileges
  security_opt:
    - apparmor=unconfined
    - seccomp=unconfined

x-all-privileges-trait: &with-all-privileges
  privileged: true
  cap_add:
    - ALL

x-network-privileges-trait: &with-network-privileges
  cap_add:
    - NET_ADMIN
    - SYS_ADMIN
    - SYS_RESOURCE

x-base-service-definition: &base-service
  restart: 'unless-stopped'
  # for docker-compose only, no effect on balenaCloud
  env_file:
    - .env
  tty: true  # send syastemd logs from containers to stdout

services:
  # https://github.com/balena-io/open-balena-api
  api:
    <<: [
      *base-service,
      *with-default-healthcheck,
      *with-default-privileges,
      *with-default-volumes,
    ]
    image: balena/open-balena-api:v25.1.27
    depends_on:
      - db
      - redis
      - s3
    environment:
      <<: *default-environment
      CONTRACTS_PUBLIC_REPO_NAME: contracts
      CONTRACTS_PUBLIC_REPO_OWNER: balena-io
      DB_GENERAL_REPLICA_MAX_USES: 1000
      DB_GENERAL_REPLICA_PORT: 5432
      DB_HOST: db
      DB_PORT: 5432
      DB_STATE_REPLICA_MAX_USES: 1000
      DB_STATE_REPLICA_PORT: 5432
      DB_USER: docker
      HOSTS_CONFIG: API_HOST:api,DB_HOST:db,DELTA_HOST:delta,HOST:api,REDIS_HOST:redis,TOKEN_AUTH_CERT_ISSUER:api,VPN_HOST:cloudlink,REGISTRY2_HOST:registry2
      IMAGE_STORAGE_BUCKET: resin-production-img-cloudformation
      IMAGE_STORAGE_ENDPOINT: s3.amazonaws.com
      IMAGE_STORAGE_PREFIX: images
      JSON_WEB_TOKEN_EXPIRY_MINUTES: 10080
      NUM_WORKERS: 1
      OAUTH_CALLBACK_PROTOCOL: https
      PORT: 80
      REDIS_HOST: redis:6379
      REDIS_IS_CLUSTER: 'false'
      TOKEN_AUTH_JWT_ALGO: ES256
      TOKENS_CONFIG: API_SERVICE_API_KEY:hex,AUTH_RESINOS_REGISTRY_CODE:hex,COOKIE_SESSION_SECRET:hex,JSON_WEB_TOKEN_SECRET:hex,MIXPANEL_TOKEN:hex,SUPERUSER_PASSWORD:hex,TOKEN_AUTH_BUILDER_TOKEN:hex,VPN_GUEST_API_KEY:hex,VPN_SERVICE_API_KEY:hex,API_VPN_SERVICE_API_KEY:API_SERVICE_API_KEY,REGISTRY2_TOKEN:TOKEN_AUTH_BUILDER_TOKEN
      TRUST_PROXY: 172.16.0.0/12
      VPN_PORT: 443
      WEBRESOURCES_S3_BUCKET: web-resources
      WEBRESOURCES_S3_REGION: 'us-east-1'  # this is required for minio

  # https://github.com/balena-io/open-balena-registry
  registry:
    <<: [
      *base-service,
      *with-default-healthcheck,
      *with-default-privileges,
    ]
    image: balena/open-balena-registry:v2.39.60
    volumes:
      - certs-data:/certs
      - resin-data:/balena
    depends_on:
      - redis
      - s3
    environment:
      COMMON_REGION: open-balena
      HOSTS_CONFIG: REGISTRY2_HOST:registry2,REGISTRY2_TOKEN_AUTH_ISSUER:api,REGISTRY2_TOKEN_AUTH_REALM:api
      REGISTRY2_CACHE_ADDR: redis:6379
      REGISTRY2_CACHE_DB: 1
      REGISTRY2_CACHE_ENABLED: 'true'
      REGISTRY2_S3_BUCKET: registry-data
      REGISTRY2_STORAGEPATH: /data
      TOKENS_CONFIG: REGISTRY2_SECRETKEY:hex

  # https://github.com/balena-io/open-balena-vpn
  vpn:
    <<: [
      *base-service,
      *with-default-healthcheck,
      *with-default-volumes,
      # privileges in order from minimum to maximum
      *with-network-privileges,
      *with-default-privileges,
    ]
    image: balena/open-balena-vpn:v11.30.22
    depends_on:
      - api
    environment:
      HOSTS_CONFIG: VPN_HOST:cloudlink
      TOKENS_CONFIG: ','
      VPN_HAPROXY_USEPROXYPROTOCOL: 'true'
      VPN_PORT: 443
      # ensure correct service instance IP is registered with the API
      VPN_SERVICE_REGISTER_INTERFACE: eth0

  # https://github.com/balena-io/open-balena-db
  db:
    <<: *base-service
    image: balena/open-balena-db:v5.2.2
    volumes:
      - db-data:/var/lib/postgresql/data
    environment:
      <<: *default-environment
    healthcheck:
      test: pg_isready -U "$${DB_USER}" -d "$${DB_NAME}"

  # https://github.com/balena-io/open-balena-s3
  s3:
    <<: [
      *base-service,
      *with-default-healthcheck,
      *with-default-privileges,
    ]
    image: balena/open-balena-s3:v2.28.47
    volumes:
      - s3-data:/export
      - certs-data:/certs
      - resin-data:/balena
    environment:
      BUCKETS: registry-data;web-resources
      HOSTS_CONFIG: REGISTRY2_S3_REGION_ENDPOINT:s3,WEBRESOURCES_S3_HOST:s3
      TOKENS_CONFIG: REGISTRY2_S3_KEY:hex,REGISTRY2_S3_SECRET:hex,S3_MINIO_ACCESS_KEY:REGISTRY2_S3_KEY,S3_MINIO_SECRET_KEY:REGISTRY2_S3_SECRET,WEBRESOURCES_S3_ACCESS_KEY:REGISTRY2_S3_KEY,WEBRESOURCES_S3_SECRET_KEY:REGISTRY2_S3_SECRET

  # https://hub.docker.com/_/redis
  redis:
    <<: *base-service
    # https://redis.io/blog/what-redis-license-change-means-for-our-managed-service-providers/
    image: redis:7.2-alpine
    volumes:
      - redis-data:/data
    healthcheck:
      <<: *default-healthcheck
      test: echo INFO | redis-cli | grep redis_version

  # https://github.com/balena-io/open-balena-haproxy
  haproxy:
    <<: [
      *base-service,
      *with-default-privileges,
      *with-default-volumes,
    ]
    build: src/haproxy
    sysctls:
      # https://github.com/docker-library/haproxy/issues/160
      net.ipv4.ip_unprivileged_port_start: 0
    healthcheck:
      <<: *default-healthcheck
      test: true | openssl s_client -connect localhost:443
    ports:
      # haproxy/http
      - '80:80/tcp'
      # haproxy/tcp-router
      - '443:443/tcp'
      # haproxy/stats
      - '1936:1936/tcp'
    environment:
      LOGLEVEL: info

  # dynamically configure Docker network aliases based on DNS_TLD and ALIAS list
  # allows DNS resolution from systemd-less images on the Docker network
  haproxy-sidecar:
    <<: *base-service
    build: src/haproxy-sidecar
    volumes:
      - /var/run/docker.sock:/host/run/docker.sock
    environment:
      DOCKER_HOST: unix:///host/run/docker.sock
      # resolved internally as {{service}}.{{dns-tld-without-balena-device-uuid}} to haproxy service
      ALIASES: api,ca,cloudlink,db,delta,logs,redis,registry2,s3,stats,tunnel
    labels:
      io.balena.features.balena-socket: 1
      io.balena.features.supervisor-api	: 1

  # https://github.com/balena-io/cert-manager
  # https://certbot.eff.org/docs/using.html
  # https://certbot-dns-cloudflare.readthedocs.io/
  cert-manager:
    <<: *base-service
    build: src/cert-manager
    volumes:
      - cert-manager-data:/etc/letsencrypt
      - certs-data:/certs
      - resin-data:/balena
    depends_on:
      - balena-ca
    environment:
      # wildcard certificate for reverse proxy
      SSH_KEY_NAMES: ','
      SUBJECT_ALTERNATE_NAMES: '*'
    labels:
      io.balena.features.balena-api: 1
      io.balena.features.supervisor-api: 1

  # https://github.com/balena-io/ca-private
  # https://github.com/cloudflare/cfssl/blob/master/doc/api/intro.txt
  balena-ca:
    <<: *base-service
    image: balena/ca-private:v0.0.14
    volumes:
      - pki-data:/pki
      - certs-data:/certs
      - resin-data:/balena
    healthcheck:
      test: curl --silent -I --fail localhost:8888
      interval: 60s
      timeout: 60s
      retries: 10
    labels:
      # future expansion
      io.balena.features.balena-api: 1
      io.balena.features.supervisor-api: 1



  # --- the following are not required for runtime operation of openBalena

  # only relevant when running in AWS/EC2
  tag-sidecar:
    build: src/tag-sidecar
    restart: 'no'
    environment:
      ENABLED: 'true'
    labels:
      io.balena.features.balena-api: 1

  # Software Under Test (SUT) tests orchestrator
  sut:
    <<: [
      *base-service,
      *with-extended-privileges,
      *with-network-privileges,
    ]
    build: src/balena-tests
    command: /usr/sbin/balena.sh
    environment:
      DOCKER_CERT_PATH: /docker-pki/client
      DOCKER_HOST: docker:2376
      DOCKER_TLS_VERIFY: 'true'
      GUEST_IMAGE: /balena/balena.img
    volumes:
      - builder-certs-client:/docker-pki/client
      - certs-data:/certs
      - resin-data:/balena
    labels:
      io.balena.features.balena-api: 1
      io.balena.features.supervisor-api: 1
    restart: 'no'

  # virtual Device Under Test (DUT)
  dut:
    <<: [
      *base-service,
      *with-extended-privileges,
      *with-network-privileges,
    ]
    # https://hub.docker.com/r/qemux/qemu-docker
    # https://github.com/qemus/qemu-docker
    build: src/test-device
    entrypoint:
      - /bin/sh
      - -c
    command:
      - /usr/sbin/balena.sh
    environment:
      GUEST_IMAGE: /balena/balena.img
      MEMORY: 3072M
      CPU: 4
    volumes:
      - resin-data:/balena
    devices:
      - /dev/net/tun
    restart: 'no'

  # https://hub.docker.com/_/docker
  # pseudo(builder) service for balena-tests
  docker:
    <<: [
      *base-service,
      *with-extended-privileges,
      *with-network-privileges,
    ]
    image: docker:dind
    entrypoint:
      - /bin/sh
      - -c
    command:
      - |
        set -x

        cp /certs/root-ca.pem /certs/server-ca.pem /usr/local/share/ca-certificates/ \
          && update-ca-certificates

        exec /usr/local/bin/dockerd-entrypoint.sh
    volumes:
      - /sys:/sys
      - builder-certs-ca:/docker-pki/ca
      - builder-certs-client:/docker-pki/client
      - builder-data:/var/lib/docker
      - certs-data:/certs
    environment:
      DOCKER_TLS_CERTDIR: /docker-pki
    healthcheck:
      test: docker system info
      interval: 60s
      timeout: 60s
      retries: 5
    labels:
      io.balena.features.sysfs: 1