Merge pull request #1274 from exarkun/3994.refactor-circleci-configuration

Fixes: ticket:3994

.circleci/config.yml

@@ -11,19 +11,31 @@
 #
 version: 2.1
 
-# A template that can be shared between the two different image-building
+# Every job that pushes a Docker image from Docker Hub must authenticate to
+# it.  Define a couple yaml anchors that can be used to supply the necessary
+# credentials.
+
+# First is a CircleCI job context which makes Docker Hub credentials available
+# in the environment.
+#
+# Contexts are managed in the CircleCI web interface:
+#
+#  https://app.circleci.com/settings/organization/github/tahoe-lafs/contexts
+dockerhub-context-template: &DOCKERHUB_CONTEXT
+  context: "dockerhub-auth"
+
+# Next is a Docker executor template that gets the credentials from the
+# environment and supplies them to the executor.
+dockerhub-auth-template: &DOCKERHUB_AUTH
+  - auth:
+      username: $DOCKERHUB_USERNAME
+      password: $DOCKERHUB_PASSWORD
+
+  # A template that can be shared between the two different image-building
 # workflows.
 .images: &IMAGES
   jobs:
-    # Every job that pushes a Docker image from Docker Hub needs to provide
+    - "build-image-debian-11":
-    # credentials.  Use this first job to define a yaml anchor that can be
-    # used to supply a CircleCI job context which makes Docker Hub credentials
-    # available in the environment.
-    #
-    # Contexts are managed in the CircleCI web interface:
-    #
-    #  https://app.circleci.com/settings/organization/github/tahoe-lafs/contexts
-    - "build-image-debian-11": &DOCKERHUB_CONTEXT
         <<: *DOCKERHUB_CONTEXT
     - "build-image-ubuntu-20-04":
         <<: *DOCKERHUB_CONTEXT
@@ -71,12 +83,20 @@ workflows:
           {}
 
       - "nixos":
-          name: "NixOS 22.11"
+          name: "<<matrix.pythonVersion>>"
           nixpkgs: "22.11"
+          matrix:
+            parameters:
+              pythonVersion:
+                - "python310"
 
       - "nixos":
-          name: "NixOS unstable"
+          name: "<<matrix.pythonVersion>>"
           nixpkgs: "unstable"
+          matrix:
+            parameters:
+              pythonVersion:
+                - "python311"
 
       # Eventually, test against PyPy 3.8
       #- "pypy27-buster":
@@ -113,30 +133,7 @@ workflows:
     # Build as part of the workflow but only if requested.
     when: "<< pipeline.parameters.build-images >>"
 
-
 jobs:
-  dockerhub-auth-template:
-    # This isn't a real job.  It doesn't get scheduled as part of any
-    # workflow.  Instead, it's just a place we can hang a yaml anchor to
-    # finish the Docker Hub authentication configuration.  Workflow jobs using
-    # the DOCKERHUB_CONTEXT anchor will have access to the environment
-    # variables used here.  These variables will allow the Docker Hub image
-    # pull to be authenticated and hopefully avoid hitting and rate limits.
-    docker: &DOCKERHUB_AUTH
-      - image: "null"
-        auth:
-          username: $DOCKERHUB_USERNAME
-          password: $DOCKERHUB_PASSWORD
-
-    steps:
-      - run:
-          name: "CircleCI YAML schema conformity"
-          command: |
-            # This isn't a real command.  We have to have something in this
-            # space, though, or the CircleCI yaml schema validator gets angry.
-            # Since this job is never scheduled this step is never run so the
-            # actual value here is irrelevant.
-
   codechecks:
     docker:
       - <<: *DOCKERHUB_AUTH
@@ -374,56 +371,29 @@ jobs:
           Reference the name of a niv-managed nixpkgs source (see `niv show`
           and nix/sources.json)
         type: "string"
+      pythonVersion:
+        description: >-
+          Reference the name of a Python package in nixpkgs to use.
+        type: "string"
 
-    docker:
+    executor: "nix"
-      # Run in a highly Nix-capable environment.
-      - <<: *DOCKERHUB_AUTH
-        image: "nixos/nix:2.10.3"
-
-    environment:
-      # CACHIX_AUTH_TOKEN is manually set in the CircleCI web UI and
-      # allows us to push to CACHIX_NAME.  We only need this set for
-      # `cachix use` in this step.
-      CACHIX_NAME: "tahoe-lafs-opensource"
 
     steps:
-      - "run":
+      - "nix-build":
-          # Get cachix for Nix-friendly caching.
+          nixpkgs: "<<parameters.nixpkgs>>"
-          name: "Install Basic Dependencies"
+          pythonVersion: "<<parameters.pythonVersion>>"
-          command: |
+          buildSteps:
-            NIXPKGS="https://github.com/nixos/nixpkgs/archive/nixos-<<parameters.nixpkgs>>.tar.gz"
+            - "run":
-            nix-env \
+                name: "Unit Test"
-              --file $NIXPKGS \
+                command: |
-              --install \
+                  # The dependencies are all built so we can allow more
-              -A cachix bash
+                  # parallelism here.
-              # Activate it for "binary substitution".  This sets up
+                  source .circleci/lib.sh
-              # configuration tht lets Nix download something from the cache
+                  cache_if_able nix-build \
-              # instead of building it locally, if possible.
+                    --cores 8 \
-              cachix use "${CACHIX_NAME}"
+                    --argstr pkgsVersion "nixpkgs-<<parameters.nixpkgs>>" \
-
+                    --argstr pythonVersion "<<parameters.pythonVersion>>" \
-      - "checkout"
+                    nix/tests.nix
-
-      - "run":
-          # The Nix package doesn't know how to do this part, unfortunately.
-          name: "Generate version"
-          command: |
-            nix-shell \
-              -p 'python3.withPackages (ps: [ ps.setuptools ])' \
-              --run 'python setup.py update_version'
-
-      - "run":
-          name: "Test"
-          command: |
-            # CircleCI build environment looks like it has a zillion and a
-            # half cores.  Don't let Nix autodetect this high core count
-            # because it blows up memory usage and fails the test run.  Pick a
-            # number of cores that suites the build environment we're paying
-            # for (the free one!).
-            source .circleci/lib.sh
-            cache_if_able nix-build \
-              --cores 8 \
-              --argstr pkgsVersion "nixpkgs-<<parameters.nixpkgs>>" \
-              nix/tests.nix
 
   typechecks:
     docker:
@@ -527,7 +497,6 @@ jobs:
 
   # build-image-pypy27-buster:
   #   <<: *BUILD_IMAGE
-
   #   environment:
   #     DISTRO: "pypy"
   #     TAG: "buster"
@@ -535,3 +504,87 @@ jobs:
   #     # setting up PyPy 3 in the image building toolchain.  This value is just
   #     # for constructing the right Docker image tag.
   #     PYTHON_VERSION: "2"
+
+executors:
+  nix:
+    docker:
+      # Run in a highly Nix-capable environment.
+      - <<: *DOCKERHUB_AUTH
+        image: "nixos/nix:2.10.3"
+    environment:
+      # CACHIX_AUTH_TOKEN is manually set in the CircleCI web UI and allows us
+      # to push to CACHIX_NAME.  CACHIX_NAME tells cachix which cache to push
+      # to.
+      CACHIX_NAME: "tahoe-lafs-opensource"
+
+commands:
+  nix-build:
+    parameters:
+      nixpkgs:
+        description: >-
+          Reference the name of a niv-managed nixpkgs source (see `niv show`
+          and nix/sources.json)
+        type: "string"
+      pythonVersion:
+        description: >-
+          Reference the name of a Python package in nixpkgs to use.
+        type: "string"
+      buildSteps:
+        description: >-
+          The build steps to execute after setting up the build environment.
+        type: "steps"
+
+    steps:
+      - "run":
+          # Get cachix for Nix-friendly caching.
+          name: "Install Basic Dependencies"
+          command: |
+            NIXPKGS="https://github.com/nixos/nixpkgs/archive/nixos-<<parameters.nixpkgs>>.tar.gz"
+            nix-env \
+              --file $NIXPKGS \
+              --install \
+              -A cachix bash
+              # Activate it for "binary substitution".  This sets up
+              # configuration tht lets Nix download something from the cache
+              # instead of building it locally, if possible.
+              cachix use "${CACHIX_NAME}"
+
+      - "checkout"
+
+      - "run":
+          # The Nix package doesn't know how to do this part, unfortunately.
+          name: "Generate version"
+          command: |
+            nix-shell \
+              -p 'python3.withPackages (ps: [ ps.setuptools ])' \
+              --run 'python setup.py update_version'
+
+      - "run":
+          name: "Build Dependencies"
+          command: |
+            # CircleCI build environment looks like it has a zillion and a
+            # half cores.  Don't let Nix autodetect this high core count
+            # because it blows up memory usage and fails the test run.  Pick a
+            # number of cores that suits the build environment we're paying
+            # for (the free one!).
+            source .circleci/lib.sh
+            # nix-shell will build all of the dependencies of the target but
+            # not the target itself.
+            cache_if_able nix-shell \
+              --run "" \
+              --cores 3 \
+              --argstr pkgsVersion "nixpkgs-<<parameters.nixpkgs>>" \
+              --argstr pythonVersion "<<parameters.pythonVersion>>" \
+              ./default.nix
+
+      - "run":
+          name: "Build Package"
+          command: |
+            source .circleci/lib.sh
+            cache_if_able nix-build \
+              --cores 4 \
+              --argstr pkgsVersion "nixpkgs-<<parameters.nixpkgs>>" \
+              --argstr pythonVersion "<<parameters.pythonVersion>>" \
+              ./default.nix
+
+      - steps: "<<parameters.buildSteps>>"