From 8d0869f6140f2c701aa856238c6a66243d301111 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 22 Mar 2023 09:30:52 -0400 Subject: [PATCH] Factor some shared pieces of CircleCI configuration out * Take DOCKERHUB_CONTEXT off of the single arbitrary job it was hung on and make it standalone. This isolates it from future changes to that particular job. * Take DOCKERHUB_AUTH out of `jobs` so it doesn't need a lot of extra boilerplate to pass schema validation. * Give the "nixos" job a Python version parameter so it can be instantiated multiple times to test multiple Python versions. Change the "NixOS unstable" instantiation to use Python 3.11 as a demonstration. * Move a lot of the implementation of the "nixos" job into a "nix" executor and a "nix-build" command that, together, do the generic setup required to do any nix-based builds. --- .circleci/config.yml | 215 +++++++++++++++++++++++++++---------------- 1 file changed, 135 insertions(+), 80 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ab0573a3f..d39e3de56 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -11,19 +11,30 @@ # version: 2.1 -# A template that can be shared between the two different image-building +# Every job that pushes a Docker image from Docker Hub must authenticate to +# it. Define a couple yaml anchors that can be used to supply a the necessary credentials. + +# First is a CircleCI job context which makes Docker Hub credentials available +# in the environment. +# +# Contexts are managed in the CircleCI web interface: +# +# https://app.circleci.com/settings/organization/github/tahoe-lafs/contexts +dockerhub-context-template: &DOCKERHUB_CONTEXT + context: "dockerhub-auth" + +# Next is a Docker executor template that gets the credentials from the +# environment and supplies them to the executor. +dockerhub-auth-template: &DOCKERHUB_AUTH + - auth: + username: $DOCKERHUB_USERNAME + password: $DOCKERHUB_PASSWORD + + # A template that can be shared between the two different image-building # workflows. .images: &IMAGES jobs: - # Every job that pushes a Docker image from Docker Hub needs to provide - # credentials. Use this first job to define a yaml anchor that can be - # used to supply a CircleCI job context which makes Docker Hub credentials - # available in the environment. - # - # Contexts are managed in the CircleCI web interface: - # - # https://app.circleci.com/settings/organization/github/tahoe-lafs/contexts - - "build-image-debian-11": &DOCKERHUB_CONTEXT + - "build-image-debian-11": <<: *DOCKERHUB_CONTEXT - "build-image-ubuntu-20-04": <<: *DOCKERHUB_CONTEXT @@ -71,12 +82,20 @@ workflows: {} - "nixos": - name: "NixOS 22.11" + name: "<>" nixpkgs: "22.11" + matrix: + parameters: + pythonVersion: + - "python310" - "nixos": - name: "NixOS unstable" + name: "<>" nixpkgs: "unstable" + matrix: + parameters: + pythonVersion: + - "python311" # Eventually, test against PyPy 3.8 #- "pypy27-buster": @@ -113,30 +132,7 @@ workflows: # Build as part of the workflow but only if requested. when: "<< pipeline.parameters.build-images >>" - jobs: - dockerhub-auth-template: - # This isn't a real job. It doesn't get scheduled as part of any - # workflow. Instead, it's just a place we can hang a yaml anchor to - # finish the Docker Hub authentication configuration. Workflow jobs using - # the DOCKERHUB_CONTEXT anchor will have access to the environment - # variables used here. These variables will allow the Docker Hub image - # pull to be authenticated and hopefully avoid hitting and rate limits. - docker: &DOCKERHUB_AUTH - - image: "null" - auth: - username: $DOCKERHUB_USERNAME - password: $DOCKERHUB_PASSWORD - - steps: - - run: - name: "CircleCI YAML schema conformity" - command: | - # This isn't a real command. We have to have something in this - # space, though, or the CircleCI yaml schema validator gets angry. - # Since this job is never scheduled this step is never run so the - # actual value here is irrelevant. - codechecks: docker: - <<: *DOCKERHUB_AUTH @@ -374,56 +370,32 @@ jobs: Reference the name of a niv-managed nixpkgs source (see `niv show` and nix/sources.json) type: "string" + pythonVersion: + description: >- + Reference the name of a Python package in nixpkgs to use. + type: "string" - docker: - # Run in a highly Nix-capable environment. - - <<: *DOCKERHUB_AUTH - image: "nixos/nix:2.10.3" - - environment: - # CACHIX_AUTH_TOKEN is manually set in the CircleCI web UI and - # allows us to push to CACHIX_NAME. We only need this set for - # `cachix use` in this step. - CACHIX_NAME: "tahoe-lafs-opensource" + executor: "nix" steps: - - "run": - # Get cachix for Nix-friendly caching. - name: "Install Basic Dependencies" - command: | - NIXPKGS="https://github.com/nixos/nixpkgs/archive/nixos-<>.tar.gz" - nix-env \ - --file $NIXPKGS \ - --install \ - -A cachix bash - # Activate it for "binary substitution". This sets up - # configuration tht lets Nix download something from the cache - # instead of building it locally, if possible. - cachix use "${CACHIX_NAME}" + - "nix-build": + nixpkgs: "<>" + pythonVersion: "<>" + buildSteps: + - "run": + name: "Unit Test" + command: | + # The dependencies are all built so we can allow more + # parallelism here. + source .circleci/lib.sh + cache_if_able nix-build \ + --cores 8 \ + --argstr pkgsVersion "nixpkgs-<>" \ + --argstr pythonVersion "<>" \ + nix/unittests.nix - - "checkout" - - "run": - # The Nix package doesn't know how to do this part, unfortunately. - name: "Generate version" - command: | - nix-shell \ - -p 'python3.withPackages (ps: [ ps.setuptools ])' \ - --run 'python setup.py update_version' - - "run": - name: "Test" - command: | - # CircleCI build environment looks like it has a zillion and a - # half cores. Don't let Nix autodetect this high core count - # because it blows up memory usage and fails the test run. Pick a - # number of cores that suites the build environment we're paying - # for (the free one!). - source .circleci/lib.sh - cache_if_able nix-build \ - --cores 8 \ - --argstr pkgsVersion "nixpkgs-<>" \ - nix/tests.nix typechecks: docker: @@ -527,7 +499,6 @@ jobs: # build-image-pypy27-buster: # <<: *BUILD_IMAGE - # environment: # DISTRO: "pypy" # TAG: "buster" @@ -535,3 +506,87 @@ jobs: # # setting up PyPy 3 in the image building toolchain. This value is just # # for constructing the right Docker image tag. # PYTHON_VERSION: "2" + +executors: + nix: + docker: + # Run in a highly Nix-capable environment. + - <<: *DOCKERHUB_AUTH + image: "nixos/nix:2.10.3" + environment: + # CACHIX_AUTH_TOKEN is manually set in the CircleCI web UI and + # allows us to push to CACHIX_NAME. We only need this set for + # `cachix use` in this step. + CACHIX_NAME: "tahoe-lafs-opensource" + +commands: + nix-build: + parameters: + nixpkgs: + description: >- + Reference the name of a niv-managed nixpkgs source (see `niv show` + and nix/sources.json) + type: "string" + pythonVersion: + description: >- + Reference the name of a Python package in nixpkgs to use. + type: "string" + buildSteps: + description: >- + The build steps to execute after setting up the build environment. + type: "steps" + + steps: + - "run": + # Get cachix for Nix-friendly caching. + name: "Install Basic Dependencies" + command: | + NIXPKGS="https://github.com/nixos/nixpkgs/archive/nixos-<>.tar.gz" + nix-env \ + --file $NIXPKGS \ + --install \ + -A cachix bash + # Activate it for "binary substitution". This sets up + # configuration tht lets Nix download something from the cache + # instead of building it locally, if possible. + cachix use "${CACHIX_NAME}" + + - "checkout" + + - "run": + # The Nix package doesn't know how to do this part, unfortunately. + name: "Generate version" + command: | + nix-shell \ + -p 'python3.withPackages (ps: [ ps.setuptools ])' \ + --run 'python setup.py update_version' + + - "run": + name: "Build Dependencies" + command: | + # CircleCI build environment looks like it has a zillion and a + # half cores. Don't let Nix autodetect this high core count + # because it blows up memory usage and fails the test run. Pick a + # number of cores that suits the build environment we're paying + # for (the free one!). + source .circleci/lib.sh + # nix-shell will build all of the dependencies of the target but + # not the target itself. + cache_if_able nix-shell \ + --run "" \ + --cores 3 \ + --argstr pkgsVersion "nixpkgs-<>" \ + --argstr pythonVersion "<>" \ + ./default.nix + + - "run": + name: "Build Package" + command: | + source .circleci/lib.sh + cache_if_able nix-build \ + --cores 4 \ + --argstr pkgsVersion "nixpkgs-<>" \ + --argstr pythonVersion "<>" \ + ./default.nix + + - steps: "<>"