From 78e04cc82170f8139b67b419f6cc72e3e75bc477 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 21 Dec 2022 06:25:22 -0500 Subject: [PATCH 1/4] Modernize cachix usage; attempt to fix CircleCI conditional CIRCLE_PR_NUMBER documentation may just be wrong. It seems like maybe it is never set? Try inspecting the source repo value instead. --- .circleci/config.yml | 73 ++++++++++++-------------------------------- .circleci/lib.sh | 25 +++++++++++++++ 2 files changed, 44 insertions(+), 54 deletions(-) create mode 100644 .circleci/lib.sh diff --git a/.circleci/config.yml b/.circleci/config.yml index d7e4f2563..4dcf2a2db 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -380,7 +380,7 @@ jobs: docker: # Run in a highly Nix-capable environment. - <<: *DOCKERHUB_AUTH - image: "nixos/nix:2.3.16" + image: "nixos/nix:2.10.3" environment: # CACHIX_AUTH_TOKEN is manually set in the CircleCI web UI and @@ -390,27 +390,21 @@ jobs: steps: - "run": - # The nixos/nix image does not include ssh. Install it so the - # `checkout` step will succeed. We also want cachix for - # Nix-friendly caching. + # Get cachix for Nix-friendly caching. name: "Install Basic Dependencies" command: | + NIXPKGS="https://github.com/nixos/nixpkgs/archive/nixos-<>.tar.gz" nix-env \ - --file https://github.com/nixos/nixpkgs/archive/nixos-<>.tar.gz \ + --file $NIXPKGS \ --install \ - -A openssh cachix bash + -A cachix bash + # Activate it for "binary substitution". This sets up + # configuration tht lets Nix download something from the cache + # instead of building it locally, if possible. + cachix use "${CACHIX_NAME}" - "checkout" - - run: - name: "Cachix setup" - # Record the store paths that exist before we did much. There's no - # reason to cache these, they're either in the image or have to be - # retrieved before we can use cachix to restore from cache. - command: | - cachix use "${CACHIX_NAME}" - nix path-info --all > /tmp/store-path-pre-build - - "run": # The Nix package doesn't know how to do this part, unfortunately. name: "Generate version" @@ -432,50 +426,21 @@ jobs: # build a couple simple little dependencies that don't take # advantage of multiple cores and we get a little speedup by doing # them in parallel. - nix-build --cores 3 --max-jobs 2 --argstr pkgsVersion "nixpkgs-<>" + source .circleci/lib.sh + cache_if_able nix-build \ + --cores 3 \ + --max-jobs 2 \ + --argstr pkgsVersion "nixpkgs-<>" - "run": name: "Test" command: | # Let it go somewhat wild for the test suite itself - nix-build --cores 8 --argstr pkgsVersion "nixpkgs-<>" tests.nix - - - run: - # Send any new store objects to cachix. - name: "Push to Cachix" - when: "always" - command: | - # Cribbed from - # https://circleci.com/blog/managing-secrets-when-you-have-pull-requests-from-outside-contributors/ - if [ -n "$CIRCLE_PR_NUMBER" ]; then - # I'm sure you're thinking "CIRCLE_PR_NUMBER must just be the - # number of the PR being built". Sorry, dear reader, you have - # guessed poorly. It is also conditionally set based on whether - # this is a PR from a fork or not. - # - # https://circleci.com/docs/2.0/env-vars/#built-in-environment-variables - echo "Skipping Cachix push for forked PR." - else - # If this *isn't* a build from a fork then we have the Cachix - # write key in our environment and we can push any new objects - # to Cachix. - # - # To decide what to push, we inspect the list of store objects - # that existed before and after we did most of our work. Any - # that are new after the work is probably a useful thing to have - # around so push it to the cache. We exclude all derivation - # objects (.drv files) because they're cheap to reconstruct and - # by the time you know their cache key you've already done all - # the work anyway. - # - # This shell expression for finding the objects and pushing them - # was from the Cachix docs: - # - # https://docs.cachix.org/continuous-integration-setup/circleci.html - # - # but they seem to have removed it now. - bash -c "comm -13 <(sort /tmp/store-path-pre-build | grep -v '\.drv$') <(nix path-info --all | grep -v '\.drv$' | sort) | cachix push $CACHIX_NAME" - fi + source .circleci/lib.sh + cache_if_able nix-build \ + --cores 8 \ + --argstr pkgsVersion "nixpkgs-<>" \ + tests.nix typechecks: docker: diff --git a/.circleci/lib.sh b/.circleci/lib.sh new file mode 100644 index 000000000..f3fe07bae --- /dev/null +++ b/.circleci/lib.sh @@ -0,0 +1,25 @@ +# Run a command, enabling cache writes to cachix if possible. The command is +# accepted as a variable number of positional arguments (like argv). +function cache_if_able() { + # The `cachix watch-exec ...` does our cache population. When it sees + # something added to the store (I guess) it pushes it to the named cache. + # + # We can only *push* to it if we have a CACHIX_AUTH_TOKEN, though. + # in-repo jobs will get this from CircleCI configuration but jobs from + # forks may not. + if [ -v CACHIX_AUTH_TOKEN ]; then + echo "Cachix credentials present; will attempt to write to cache." + cachix watch-exec "${CACHIX_NAME}" -- "$@" + else + # If we're building a from a forked repository then we're allowed to + # not have the credentials (but it's also fine if the owner of the + # fork supplied their own). + if [ "${CIRCLE_PR_REPONAME}" == "https://github.com/tahoe-lafs/tahoe-lafs" ]; then + echo "Required credentials (CACHIX_AUTH_TOKEN) are missing." + return 1 + else + echo "Cachix credentials missing; will not attempt cache writes." + "$@" + fi + fi +} From 21af00bf83ff8b1f684d965d772c564d7af92e2b Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 21 Dec 2022 06:27:41 -0500 Subject: [PATCH 2/4] Report the CIRCLE_PR_REPONAME too, because who knows --- .circleci/lib.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/lib.sh b/.circleci/lib.sh index f3fe07bae..cc7ce5e97 100644 --- a/.circleci/lib.sh +++ b/.circleci/lib.sh @@ -7,6 +7,7 @@ function cache_if_able() { # We can only *push* to it if we have a CACHIX_AUTH_TOKEN, though. # in-repo jobs will get this from CircleCI configuration but jobs from # forks may not. + echo "Building PR from repo: ${CIRCLE_PR_REPONAME}" if [ -v CACHIX_AUTH_TOKEN ]; then echo "Cachix credentials present; will attempt to write to cache." cachix watch-exec "${CACHIX_NAME}" -- "$@" From 25eb3ca262e0a2bff842e8eff78284f0723faa42 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 21 Dec 2022 06:47:21 -0500 Subject: [PATCH 3/4] Switch to a variable observed in practice There is apparently no CIRCLE_PR_REPONAME set in the runtime environment, either, despite what the docs say. --- .circleci/lib.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/lib.sh b/.circleci/lib.sh index cc7ce5e97..7717cdb18 100644 --- a/.circleci/lib.sh +++ b/.circleci/lib.sh @@ -7,7 +7,7 @@ function cache_if_able() { # We can only *push* to it if we have a CACHIX_AUTH_TOKEN, though. # in-repo jobs will get this from CircleCI configuration but jobs from # forks may not. - echo "Building PR from repo: ${CIRCLE_PR_REPONAME}" + echo "Building PR from user/org: ${CIRCLE_PROJECT_USERNAME}" if [ -v CACHIX_AUTH_TOKEN ]; then echo "Cachix credentials present; will attempt to write to cache." cachix watch-exec "${CACHIX_NAME}" -- "$@" @@ -15,7 +15,7 @@ function cache_if_able() { # If we're building a from a forked repository then we're allowed to # not have the credentials (but it's also fine if the owner of the # fork supplied their own). - if [ "${CIRCLE_PR_REPONAME}" == "https://github.com/tahoe-lafs/tahoe-lafs" ]; then + if [ "${CIRCLE_PROJECT_USERNAME}" == "tahoe-lafs" ]; then echo "Required credentials (CACHIX_AUTH_TOKEN) are missing." return 1 else From 2da3d43b2e4e7a0b6dff7f2efd7a8bb675a00ced Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 21 Dec 2022 07:22:37 -0500 Subject: [PATCH 4/4] news fragment --- newsfragments/3870.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3870.minor diff --git a/newsfragments/3870.minor b/newsfragments/3870.minor new file mode 100644 index 000000000..e69de29bb