From 08b594b8be7437b53c7ad71999f130a6a11b54d8 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 7 Aug 2023 10:18:45 -0400 Subject: [PATCH 01/75] Declare the Windows orb for easier Windows environment setup --- .circleci/config.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 0c831af04..6b2d84d92 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -11,6 +11,10 @@ # version: 2.1 +orbs: + # Pull in CircleCI support for a Windows executor + windows: "circleci/windows@5.0.0" + # Every job that pushes a Docker image from Docker Hub must authenticate to # it. Define a couple yaml anchors that can be used to supply the necessary # credentials. From 814ba4c88b58c088b5445420ba35de3ed18f1477 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 7 Aug 2023 10:19:07 -0400 Subject: [PATCH 02/75] Add Windows executor holding the Windows test environment config --- .circleci/config.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 6b2d84d92..2375368ca 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -531,6 +531,12 @@ jobs: # PYTHON_VERSION: "2" executors: + windows: + # Choose a Windows environment that closest matches our testing + # requirements and goals. + # https://circleci.com/developer/orbs/orb/circleci/windows#executors-server-2022 + executor: "win/server-2022@2023.06.1" + nix: docker: # Run in a highly Nix-capable environment. From f8db7818128347d6e50dcbba93a59663597fe0d4 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 7 Aug 2023 10:19:25 -0400 Subject: [PATCH 03/75] Add a simple test job to see if the other pieces work --- .circleci/config.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 2375368ca..307d3ca69 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -116,6 +116,8 @@ workflows: - "another-locale": {} + - "windows-server-2022" + - "integration": # Run even the slow integration tests here. We need the `--` to # sneak past tox and get to pytest. @@ -137,6 +139,11 @@ workflows: when: "<< pipeline.parameters.build-images >>" jobs: + windows-server-2022: + steps: + - "run": | + Write-Host 'Hello, world.' + codechecks: docker: - <<: *DOCKERHUB_AUTH From d050faac92865a25c4b74f816f5b9796b586715f Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 7 Aug 2023 10:27:52 -0400 Subject: [PATCH 04/75] make the config match the schema --- .circleci/config.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 307d3ca69..07bdc7087 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -140,6 +140,7 @@ workflows: jobs: windows-server-2022: + executor: "windows" steps: - "run": | Write-Host 'Hello, world.' @@ -542,7 +543,8 @@ executors: # Choose a Windows environment that closest matches our testing # requirements and goals. # https://circleci.com/developer/orbs/orb/circleci/windows#executors-server-2022 - executor: "win/server-2022@2023.06.1" + machine: + image: "win/server-2022@2023.06.1" nix: docker: From a8d582237c89c7829c93cc4a744f10a63e17e177 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 7 Aug 2023 10:31:16 -0400 Subject: [PATCH 05/75] dump the useless orb --- .circleci/config.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 07bdc7087..b9866ad91 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -11,10 +11,6 @@ # version: 2.1 -orbs: - # Pull in CircleCI support for a Windows executor - windows: "circleci/windows@5.0.0" - # Every job that pushes a Docker image from Docker Hub must authenticate to # it. Define a couple yaml anchors that can be used to supply the necessary # credentials. @@ -544,7 +540,7 @@ executors: # requirements and goals. # https://circleci.com/developer/orbs/orb/circleci/windows#executors-server-2022 machine: - image: "win/server-2022@2023.06.1" + image: "windows-server-2022-gui:2023.06.1"" nix: docker: From f826914c589f7e0505a77d0cff3f6b24b2c2b669 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 7 Aug 2023 10:31:32 -0400 Subject: [PATCH 06/75] syntax --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b9866ad91..6e46d89a5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -540,7 +540,7 @@ executors: # requirements and goals. # https://circleci.com/developer/orbs/orb/circleci/windows#executors-server-2022 machine: - image: "windows-server-2022-gui:2023.06.1"" + image: "windows-server-2022-gui:2023.06.1" nix: docker: From 4abbadda47faed2d2279787fad6058a83756eb31 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 7 Aug 2023 10:32:31 -0400 Subject: [PATCH 07/75] Try a different tag https://circleci.com/developer/machine/image/windows-server-2022-gui says "2023.06.1" is a valid tag but real execution says "Job was rejected because resource class medium, image windows-server-2022-gui:2023.06.1 is not a valid resource class". Is it even complaining about the image tag? Or is it complaining about the resource class? I don't know. I didn't touch the resource class though. --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 6e46d89a5..e4c061da5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -540,7 +540,7 @@ executors: # requirements and goals. # https://circleci.com/developer/orbs/orb/circleci/windows#executors-server-2022 machine: - image: "windows-server-2022-gui:2023.06.1" + image: "windows-server-2022-gui:current" nix: docker: From bd9d2e08ef4746516d46e168db20b60b7f859dc6 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 7 Aug 2023 10:35:11 -0400 Subject: [PATCH 08/75] eh? --- .circleci/config.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e4c061da5..63bec2dde 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -136,7 +136,8 @@ workflows: jobs: windows-server-2022: - executor: "windows" + executor: + name: "windows" steps: - "run": | Write-Host 'Hello, world.' From 77c677ffc0ab957a0bb089d98f62c3eb9e78d7e3 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 7 Aug 2023 10:37:23 -0400 Subject: [PATCH 09/75] just like the example --- .circleci/config.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 63bec2dde..e1be57175 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -136,8 +136,10 @@ workflows: jobs: windows-server-2022: - executor: - name: "windows" + machine: + image: "windows-server-2022-gui:current" + shell: "powershell.exe -ExecutionPolicy Bypass" + resource_class: "windows.medium" steps: - "run": | Write-Host 'Hello, world.' From 422d4ee9ccfd16e3334a7a0ec743380cd89d1b3b Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 7 Aug 2023 10:37:55 -0400 Subject: [PATCH 10/75] previous rev started an environment, try to get a little closer to the one we want a more precise tag that won't shift around underneath us --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e1be57175..363c0ee6e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -137,7 +137,7 @@ workflows: jobs: windows-server-2022: machine: - image: "windows-server-2022-gui:current" + image: "windows-server-2022-gui:2023.06.1" shell: "powershell.exe -ExecutionPolicy Bypass" resource_class: "windows.medium" steps: From 6400a396615905ebf8fa4ac564f7d966d9d3866e Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 7 Aug 2023 10:39:13 -0400 Subject: [PATCH 11/75] so ... can we use an executor? --- .circleci/config.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 363c0ee6e..368156bb3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -136,10 +136,7 @@ workflows: jobs: windows-server-2022: - machine: - image: "windows-server-2022-gui:2023.06.1" - shell: "powershell.exe -ExecutionPolicy Bypass" - resource_class: "windows.medium" + executor: "windows" steps: - "run": | Write-Host 'Hello, world.' @@ -543,7 +540,9 @@ executors: # requirements and goals. # https://circleci.com/developer/orbs/orb/circleci/windows#executors-server-2022 machine: - image: "windows-server-2022-gui:current" + image: "windows-server-2022-gui:2023.06.1" + shell: "powershell.exe -ExecutionPolicy Bypass" + resource_class: "windows.medium" nix: docker: From d369dc0f2cdeeca255f53f5418540d3783d9f35f Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 7 Aug 2023 10:43:08 -0400 Subject: [PATCH 12/75] try to install tox --- .circleci/config.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 368156bb3..f86b46636 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -138,8 +138,10 @@ jobs: windows-server-2022: executor: "windows" steps: - - "run": | - Write-Host 'Hello, world.' + - "checkout" + + - "run": + <<: *INSTALL_TOX codechecks: docker: From e6e38128bc1ce1a6630eb95eeadcd6433662cb8e Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 7 Aug 2023 10:43:58 -0400 Subject: [PATCH 13/75] yaml syntax --- .circleci/config.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index f86b46636..e70daec41 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -135,14 +135,6 @@ workflows: when: "<< pipeline.parameters.build-images >>" jobs: - windows-server-2022: - executor: "windows" - steps: - - "checkout" - - - "run": - <<: *INSTALL_TOX - codechecks: docker: - <<: *DOCKERHUB_AUTH @@ -161,6 +153,14 @@ jobs: command: | ~/.local/bin/tox -e codechecks + windows-server-2022: + executor: "windows" + steps: + - "checkout" + + - "run": + <<: *INSTALL_TOX + pyinstaller: docker: - <<: *DOCKERHUB_AUTH From 862bda9e631770a06ffa3d9c86b717d2e2b9e0a3 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 7 Aug 2023 10:47:36 -0400 Subject: [PATCH 14/75] attempt to do something useful --- .circleci/config.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index e70daec41..5856af47d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -161,6 +161,16 @@ jobs: - "run": <<: *INSTALL_TOX + - "run": + name: "Display tool versions" + command: | + python misc/build_helpers/show-tool-versions.py + + - "run": + name: "Run Unit Tests" + command: | + python -m tox + pyinstaller: docker: - <<: *DOCKERHUB_AUTH From e2fe1af3d9303ba17e0c60b3e5d2bba43e4340a1 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 7 Aug 2023 11:03:16 -0400 Subject: [PATCH 15/75] Configure Hypothesis for the Windows job --- .circleci/config.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 5856af47d..19d70adca 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -155,6 +155,9 @@ jobs: windows-server-2022: executor: "windows" + environment: + TAHOE_LAFS_HYPOTHESIS_PROFILE: "ci" + steps: - "checkout" From 3c8a11d46822a9e7508567b970579a2d66276bf3 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 7 Aug 2023 11:12:47 -0400 Subject: [PATCH 16/75] pick a Python to support on Windows --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 19d70adca..e9e5810e5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -172,7 +172,7 @@ jobs: - "run": name: "Run Unit Tests" command: | - python -m tox + python -m tox -e py311-coverage pyinstaller: docker: From b440065952cfe9c98d65ab7728571a93aba50a6e Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 7 Aug 2023 12:59:44 -0400 Subject: [PATCH 17/75] avoid trying to call os.getuid on windows --- src/allmydata/test/cli/test_grid_manager.py | 6 ++++-- src/allmydata/test/common.py | 4 ++++ src/allmydata/test/test_client.py | 3 ++- src/allmydata/test/test_node.py | 7 ++++--- 4 files changed, 14 insertions(+), 6 deletions(-) diff --git a/src/allmydata/test/cli/test_grid_manager.py b/src/allmydata/test/cli/test_grid_manager.py index 604cd6b7b..b44b322d2 100644 --- a/src/allmydata/test/cli/test_grid_manager.py +++ b/src/allmydata/test/cli/test_grid_manager.py @@ -23,6 +23,9 @@ import click.testing from ..common_util import ( run_cli, ) +from ..common import ( + superuser, +) from twisted.internet.defer import ( inlineCallbacks, ) @@ -34,7 +37,6 @@ from twisted.python.runtime import ( ) from allmydata.util import jsonbytes as json - class GridManagerCommandLine(TestCase): """ Test the mechanics of the `grid-manager` command @@ -223,7 +225,7 @@ class GridManagerCommandLine(TestCase): ) @skipIf(platform.isWindows(), "We don't know how to set permissions on Windows.") - @skipIf(os.getuid() == 0, "cannot test as superuser with all permissions") + @skipIf(superuser, "cannot test as superuser with all permissions") def test_sign_bad_perms(self): """ Error reported if we can't create certificate file diff --git a/src/allmydata/test/common.py b/src/allmydata/test/common.py index db2921e86..d61bc28f1 100644 --- a/src/allmydata/test/common.py +++ b/src/allmydata/test/common.py @@ -117,6 +117,10 @@ from subprocess import ( PIPE, ) +# Is the process running as an OS user with elevated privileges (ie, root)? +# We only know how to determine this for POSIX systems. +superuser = getattr(os, "getuid", lambda: -1)() == 0 + EMPTY_CLIENT_CONFIG = config_from_string( "/dev/null", "tub.port", diff --git a/src/allmydata/test/test_client.py b/src/allmydata/test/test_client.py index 86c95a310..c0cce2809 100644 --- a/src/allmydata/test/test_client.py +++ b/src/allmydata/test/test_client.py @@ -77,6 +77,7 @@ from allmydata.scripts.common import ( from foolscap.api import flushEventualQueue import allmydata.test.common_util as testutil from .common import ( + superuser, EMPTY_CLIENT_CONFIG, SyncTestCase, AsyncBrokenTestCase, @@ -151,7 +152,7 @@ class Basic(testutil.ReallyEqualMixin, unittest.TestCase): # EnvironmentError when reading a file that really exists), on # windows, please fix this @skipIf(platform.isWindows(), "We don't know how to set permissions on Windows.") - @skipIf(os.getuid() == 0, "cannot test as superuser with all permissions") + @skipIf(superuser, "cannot test as superuser with all permissions") def test_unreadable_config(self): basedir = "test_client.Basic.test_unreadable_config" os.mkdir(basedir) diff --git a/src/allmydata/test/test_node.py b/src/allmydata/test/test_node.py index 1469ec5b2..90da877fb 100644 --- a/src/allmydata/test/test_node.py +++ b/src/allmydata/test/test_node.py @@ -62,6 +62,7 @@ from .common import ( ConstantAddresses, SameProcessStreamEndpointAssigner, UseNode, + superuser, ) def port_numbers(): @@ -325,7 +326,7 @@ class TestCase(testutil.SignalMixin, unittest.TestCase): self.assertEqual(config.items("nosuch", default), default) @skipIf(platform.isWindows(), "We don't know how to set permissions on Windows.") - @skipIf(os.getuid() == 0, "cannot test as superuser with all permissions") + @skipIf(superuser, "cannot test as superuser with all permissions") def test_private_config_unreadable(self): """ Asking for inaccessible private config is an error @@ -341,7 +342,7 @@ class TestCase(testutil.SignalMixin, unittest.TestCase): config.get_or_create_private_config("foo") @skipIf(platform.isWindows(), "We don't know how to set permissions on Windows.") - @skipIf(os.getuid() == 0, "cannot test as superuser with all permissions") + @skipIf(superuser, "cannot test as superuser with all permissions") def test_private_config_unreadable_preexisting(self): """ error if reading private config data fails @@ -398,7 +399,7 @@ class TestCase(testutil.SignalMixin, unittest.TestCase): self.assertEqual(len(counter), 1) # don't call unless necessary self.assertEqual(value, "newer") - @skipIf(os.getuid() == 0, "cannot test as superuser with all permissions") + @skipIf(superuser, "cannot test as superuser with all permissions") def test_write_config_unwritable_file(self): """ Existing behavior merely logs any errors upon writing From e92e7faeea00c1dacef75d36bf1815b5bdf296ad Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 7 Aug 2023 15:25:58 -0400 Subject: [PATCH 18/75] how much difference does this make --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e9e5810e5..d204a5531 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -557,7 +557,7 @@ executors: machine: image: "windows-server-2022-gui:2023.06.1" shell: "powershell.exe -ExecutionPolicy Bypass" - resource_class: "windows.medium" + resource_class: "windows.large" nix: docker: From 9f5173e7302bb1258f64e03b5f8506587a816a4f Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 7 Aug 2023 15:58:33 -0400 Subject: [PATCH 19/75] attempt to report coverage results to coveralls --- .circleci/config.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index d204a5531..431b2c70a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -174,6 +174,12 @@ jobs: command: | python -m tox -e py311-coverage + - "run": + name: "Upload Coverage" + command: | + python -m pip install coveralls + python -m coveralls + pyinstaller: docker: - <<: *DOCKERHUB_AUTH From f649968ab5442dc1fa76554084cae99fca660a24 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 7 Aug 2023 16:21:27 -0400 Subject: [PATCH 20/75] configure the coveralls tool so it can upload the data --- .coveralls.yml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 .coveralls.yml diff --git a/.coveralls.yml b/.coveralls.yml new file mode 100644 index 000000000..1486cf5b3 --- /dev/null +++ b/.coveralls.yml @@ -0,0 +1,2 @@ +service_name: "circleci" +repo_token: "JPf16rLB7T2yjgATIxFzTsEgMdN1UNq6o" From 371f82bb4da019df5581ed0c41e7d500b902ac3f Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 7 Aug 2023 16:21:50 -0400 Subject: [PATCH 21/75] avoid problems with trial and ENOSPC I don't know if these will show up in this environment ... just copy/pasted from the GitHub Actions config. --- .circleci/config.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 431b2c70a..2f580192d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -172,7 +172,13 @@ jobs: - "run": name: "Run Unit Tests" command: | - python -m tox -e py311-coverage + # On Windows, a non-blocking pipe might respond (when emulating + # Unix-y API) with ENOSPC to indicate buffer full. Trial doesn't + # handle this well, so it breaks test runs. To attempt to solve + # this, we pipe the output through passthrough.py that will + # hopefully be able to do the right thing by using Windows APIs. + python -m pip install twisted pywin32 + python -m tox -e py311-coverage | python misc/windows-enospc/passthrough.py - "run": name: "Upload Coverage" From 4c0b72b353615499f5eb3213d65bdc7644982b8c Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 7 Aug 2023 16:38:56 -0400 Subject: [PATCH 22/75] Delightfully, this deterministically breaks in the CircleCI env --- .circleci/config.yml | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 2f580192d..431b2c70a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -172,13 +172,7 @@ jobs: - "run": name: "Run Unit Tests" command: | - # On Windows, a non-blocking pipe might respond (when emulating - # Unix-y API) with ENOSPC to indicate buffer full. Trial doesn't - # handle this well, so it breaks test runs. To attempt to solve - # this, we pipe the output through passthrough.py that will - # hopefully be able to do the right thing by using Windows APIs. - python -m pip install twisted pywin32 - python -m tox -e py311-coverage | python misc/windows-enospc/passthrough.py + python -m tox -e py311-coverage - "run": name: "Upload Coverage" From e17c8f618ea9d9a2f9fd169d75f714401121c35e Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 7 Aug 2023 16:39:52 -0400 Subject: [PATCH 23/75] run a quick subset of the tests to more quickly test the following bits --- .circleci/config.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 431b2c70a..42e59075b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -171,6 +171,8 @@ jobs: - "run": name: "Run Unit Tests" + environment: + TEST_SUITE: "allmydata.test.test_uri" command: | python -m tox -e py311-coverage From b092dd57cf62b174f731e10c231d26235e72ec1f Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 7 Aug 2023 17:02:55 -0400 Subject: [PATCH 24/75] coveralls failed to find .coveralls.yml ... also tox.ini overrides TEST_SUITE :/ Set it in the right place --- .circleci/config.yml | 5 +++-- .coveralls.yml | 2 -- tox.ini | 2 +- 3 files changed, 4 insertions(+), 5 deletions(-) delete mode 100644 .coveralls.yml diff --git a/.circleci/config.yml b/.circleci/config.yml index 42e59075b..077896d92 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -171,15 +171,16 @@ jobs: - "run": name: "Run Unit Tests" - environment: - TEST_SUITE: "allmydata.test.test_uri" command: | python -m tox -e py311-coverage - "run": name: "Upload Coverage" + environment: + COVERALLS_REPO_TOKEN: "JPf16rLB7T2yjgATIxFzTsEgMdN1UNq6o" command: | python -m pip install coveralls + python -m coveralls debug python -m coveralls pyinstaller: diff --git a/.coveralls.yml b/.coveralls.yml deleted file mode 100644 index 1486cf5b3..000000000 --- a/.coveralls.yml +++ /dev/null @@ -1,2 +0,0 @@ -service_name: "circleci" -repo_token: "JPf16rLB7T2yjgATIxFzTsEgMdN1UNq6o" diff --git a/tox.ini b/tox.ini index 67a089b0c..18d7767a2 100644 --- a/tox.ini +++ b/tox.ini @@ -55,7 +55,7 @@ extras = setenv = # Define TEST_SUITE in the environment as an aid to constructing the # correct test command below. - TEST_SUITE = allmydata + TEST_SUITE = allmydata.test.test_uri commands = # As an aid to debugging, dump all of the Python packages and their From a261c1f2025966bdc944f8023185697ed1816895 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 7 Aug 2023 17:11:54 -0400 Subject: [PATCH 25/75] try to match the paths from circleci windows for coverage path rewriting --- .coveragerc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.coveragerc b/.coveragerc index d09554cad..32b803586 100644 --- a/.coveragerc +++ b/.coveragerc @@ -19,9 +19,11 @@ skip_covered = True source = # It looks like this in the checkout src/ -# It looks like this in the Windows build environment +# It looks like this in the GitHub Actions Windows build environment D:/a/tahoe-lafs/tahoe-lafs/.tox/py*-coverage/Lib/site-packages/ # Although sometimes it looks like this instead. Also it looks like this on macOS. .tox/py*-coverage/lib/python*/site-packages/ +# And on the CircleCI Windows build envronment... + .tox/py*-coverage/Lib/site-packages/ # On some Linux CI jobs it looks like this /tmp/tahoe-lafs.tox/py*-coverage/lib/python*/site-packages/ From 5c22bf95b41958d1fbcae5362fb6deea0eed98e9 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 7 Aug 2023 17:12:06 -0400 Subject: [PATCH 26/75] maybe we don't need the debug info now --- .circleci/config.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 077896d92..fe1f5f8cb 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -180,7 +180,6 @@ jobs: COVERALLS_REPO_TOKEN: "JPf16rLB7T2yjgATIxFzTsEgMdN1UNq6o" command: | python -m pip install coveralls - python -m coveralls debug python -m coveralls pyinstaller: From 7bc1f9300f3da31f5b283b0de20f30b4d9cdd05d Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 8 Aug 2023 08:24:24 -0400 Subject: [PATCH 27/75] try to get test results loaded into circleci --- .circleci/config.yml | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index fe1f5f8cb..ede1cdb8f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -170,9 +170,17 @@ jobs: python misc/build_helpers/show-tool-versions.py - "run": - name: "Run Unit Tests" + name: "Install Dependencies" command: | - python -m tox -e py311-coverage + python -m pip install .[testenv] .[test] + + - "run": + name: "Run Unit Tests" + environment: + SUBUNITREPORTER_OUTPUT_PATH: "test-results.subunit2" + PYTHONUNBUFFERED: "1" + command: | + python -b -m coverage run -m twisted.trial --reporter=subunitv2-file --rterrors allmydata - "run": name: "Upload Coverage" @@ -182,6 +190,14 @@ jobs: python -m pip install coveralls python -m coveralls + - "run": + name: "Convert Result Log" + command: | + Get-Content -Path test-results.subunit2 -Raw | subunit2junitxml | Out-File -FilePath test-results.xml + + - "store_artifacts": + path: "test-results.xml" + pyinstaller: docker: - <<: *DOCKERHUB_AUTH From 3f37f9aee5975453a5b0d9d4f8a978c5080a2a05 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 8 Aug 2023 08:44:15 -0400 Subject: [PATCH 28/75] try to force UTF-8 to make subunitreporter work --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ede1cdb8f..1946c9346 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -180,7 +180,7 @@ jobs: SUBUNITREPORTER_OUTPUT_PATH: "test-results.subunit2" PYTHONUNBUFFERED: "1" command: | - python -b -m coverage run -m twisted.trial --reporter=subunitv2-file --rterrors allmydata + python -X utf8 -b -m coverage run -m twisted.trial --reporter=subunitv2-file --rterrors allmydata - "run": name: "Upload Coverage" From b03cf0b37b5ca2dbe443aee2c142b8d6ddd4d295 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 8 Aug 2023 09:14:06 -0400 Subject: [PATCH 29/75] send the test results to the place circleci expects for processing --- .circleci/config.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1946c9346..e8815842e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -195,6 +195,9 @@ jobs: command: | Get-Content -Path test-results.subunit2 -Raw | subunit2junitxml | Out-File -FilePath test-results.xml + - "store_test_results: + path: "test-results.xml" + - "store_artifacts": path: "test-results.xml" From 3a8480126b1813ad6eab3ac93df191176bc1a085 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 8 Aug 2023 09:19:20 -0400 Subject: [PATCH 30/75] syntax --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e8815842e..845ac3662 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -195,7 +195,7 @@ jobs: command: | Get-Content -Path test-results.subunit2 -Raw | subunit2junitxml | Out-File -FilePath test-results.xml - - "store_test_results: + - "store_test_results": path: "test-results.xml" - "store_artifacts": From 286fb206d812f62bd7af08b3332bd3f927e948df Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 8 Aug 2023 09:59:23 -0400 Subject: [PATCH 31/75] try uncorrupting the xml file --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 845ac3662..74bf3019d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -193,7 +193,7 @@ jobs: - "run": name: "Convert Result Log" command: | - Get-Content -Path test-results.subunit2 -Raw | subunit2junitxml | Out-File -FilePath test-results.xml + subunit2junitxml test-results.xml - "store_test_results": path: "test-results.xml" From ab9db4964c0f4fa671d78240d1b015c891f39379 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 8 Aug 2023 10:28:16 -0400 Subject: [PATCH 32/75] another stab --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 74bf3019d..91d46414d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -180,7 +180,7 @@ jobs: SUBUNITREPORTER_OUTPUT_PATH: "test-results.subunit2" PYTHONUNBUFFERED: "1" command: | - python -X utf8 -b -m coverage run -m twisted.trial --reporter=subunitv2-file --rterrors allmydata + python -X utf8 -b -m coverage run -m twisted.trial --reporter=subunitv2-file --rterrors allmydata.test.test_uri - "run": name: "Upload Coverage" @@ -193,7 +193,7 @@ jobs: - "run": name: "Convert Result Log" command: | - subunit2junitxml test-results.xml + Start-Process subunit2junitxml -Wait -RedirectStandardInput test-results.subunit2 -RedirectStandardOutput test-results.xml - "store_test_results": path: "test-results.xml" From 2649027c74b53499f2bc0315186a14667bac72b8 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 8 Aug 2023 10:38:16 -0400 Subject: [PATCH 33/75] back to the complete test suite --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 91d46414d..d7e576298 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -180,7 +180,7 @@ jobs: SUBUNITREPORTER_OUTPUT_PATH: "test-results.subunit2" PYTHONUNBUFFERED: "1" command: | - python -X utf8 -b -m coverage run -m twisted.trial --reporter=subunitv2-file --rterrors allmydata.test.test_uri + python -X utf8 -b -m coverage run -m twisted.trial --reporter=subunitv2-file --rterrors allmydata - "run": name: "Upload Coverage" From 208531cddc5db10c419b6509c3223ed3a297b4e2 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 8 Aug 2023 10:38:33 -0400 Subject: [PATCH 34/75] should work without `-X utf8` now --- .circleci/config.yml | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index d7e576298..13936321a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -180,7 +180,7 @@ jobs: SUBUNITREPORTER_OUTPUT_PATH: "test-results.subunit2" PYTHONUNBUFFERED: "1" command: | - python -X utf8 -b -m coverage run -m twisted.trial --reporter=subunitv2-file --rterrors allmydata + python -b -m coverage run -m twisted.trial --reporter=subunitv2-file --rterrors allmydata - "run": name: "Upload Coverage" diff --git a/setup.py b/setup.py index 86873ad53..433721d2a 100644 --- a/setup.py +++ b/setup.py @@ -413,7 +413,7 @@ setup(name="tahoe-lafs", # also set in __init__.py "pip==22.0.3", "wheel==0.37.1", "setuptools==60.9.1", - "subunitreporter==22.2.0", + "subunitreporter==23.8.0", "python-subunit==1.4.2", "junitxml==0.7", "coverage==7.2.5", From 122655842cd3dd25af9a722639a338caca0fe188 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 8 Aug 2023 11:06:31 -0400 Subject: [PATCH 35/75] try to expose the other test run artifacts --- .circleci/config.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 13936321a..4e4e183a5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -199,7 +199,10 @@ jobs: path: "test-results.xml" - "store_artifacts": - path: "test-results.xml" + path: "_trial_temp/test.log" + + - "store_artifacts": + path: "eliot.log" pyinstaller: docker: From 085a823dfdfa12a3911a6173ed8659724141af5f Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 8 Aug 2023 11:55:34 -0400 Subject: [PATCH 36/75] put back the full test suite for tox runs --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 18d7767a2..67a089b0c 100644 --- a/tox.ini +++ b/tox.ini @@ -55,7 +55,7 @@ extras = setenv = # Define TEST_SUITE in the environment as an aid to constructing the # correct test command below. - TEST_SUITE = allmydata.test.test_uri + TEST_SUITE = allmydata commands = # As an aid to debugging, dump all of the Python packages and their From 23628fffd84f6a358c22fccf1ca661ac10e9cfb7 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 8 Aug 2023 12:12:45 -0400 Subject: [PATCH 37/75] Try to parameterize the Python version for Windows tests And instantiate the job with two different Python versions --- .circleci/config.yml | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4e4e183a5..4181025cc 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -112,7 +112,13 @@ workflows: - "another-locale": {} - - "windows-server-2022" + - "windows-server-2022": + name: "Windows Server 2022, Python <>" + matrix: + parameters: + pythonVersion: + - "3.9" + - "3.11" - "integration": # Run even the slow integration tests here. We need the `--` to @@ -154,6 +160,13 @@ jobs: ~/.local/bin/tox -e codechecks windows-server-2022: + parameters: + pythonVersion: + description: >- + An argument to pass to the `py` launcher to choose a Python version. + type: "string" + default: "" + executor: "windows" environment: TAHOE_LAFS_HYPOTHESIS_PROFILE: "ci" @@ -167,12 +180,12 @@ jobs: - "run": name: "Display tool versions" command: | - python misc/build_helpers/show-tool-versions.py + py -<> misc/build_helpers/show-tool-versions.py - "run": name: "Install Dependencies" command: | - python -m pip install .[testenv] .[test] + py -<> -m pip install .[testenv] .[test] - "run": name: "Run Unit Tests" @@ -180,15 +193,15 @@ jobs: SUBUNITREPORTER_OUTPUT_PATH: "test-results.subunit2" PYTHONUNBUFFERED: "1" command: | - python -b -m coverage run -m twisted.trial --reporter=subunitv2-file --rterrors allmydata + py -<> -b -m coverage run -m twisted.trial --reporter=subunitv2-file --rterrors allmydata - "run": name: "Upload Coverage" environment: COVERALLS_REPO_TOKEN: "JPf16rLB7T2yjgATIxFzTsEgMdN1UNq6o" command: | - python -m pip install coveralls - python -m coveralls + py -<> -m pip install coveralls + py -<> -m coveralls - "run": name: "Convert Result Log" From 67cc25df11d8c202de4ff9addd3b9c798e492009 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 8 Aug 2023 12:13:31 -0400 Subject: [PATCH 38/75] drop Windows unit tests from GitHub Actions This drops Python 3.8 and Python 3.10 Windows coverage. --- .github/workflows/ci.yml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d3862ffad..0f38b0291 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -44,13 +44,6 @@ jobs: strategy: fail-fast: false matrix: - os: - - windows-latest - python-version: - - "3.8" - - "3.9" - - "3.10" - - "3.11" include: # On macOS don't bother with 3.8, just to get faster builds. - os: macos-12 From 8c4d99f812de8254ce610bb9451fcdae4b4edcca Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 8 Aug 2023 12:22:18 -0400 Subject: [PATCH 39/75] try to do "parallel" coveralls reporting and finish it --- .circleci/config.yml | 34 +++++++++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4181025cc..f7665cc29 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -24,6 +24,11 @@ version: 2.1 dockerhub-context-template: &DOCKERHUB_CONTEXT context: "dockerhub-auth" +# Required environment for using the coveralls tool to upload partial coverage +# reports and then finish the process. +coveralls-environment: &COVERALLS_ENVIRONMENT + COVERALLS_REPO_TOKEN: "JPf16rLB7T2yjgATIxFzTsEgMdN1UNq6o" + # Next is a Docker executor template that gets the credentials from the # environment and supplies them to the executor. dockerhub-auth-template: &DOCKERHUB_AUTH @@ -134,6 +139,11 @@ workflows: - "docs": {} + - "finish-coverage-report": + requires: + - "Windows Server 2022, Python 3.9" + - "Windows Server 2022, Python 3.11" + images: <<: *IMAGES @@ -141,6 +151,20 @@ workflows: when: "<< pipeline.parameters.build-images >>" jobs: + finish-coverage-report: + docker: + - <<: *DOCKERHUB_AUTH + image: "python:3-slim" + + steps: + - run: + name: "Indicate completion to coveralls.io" + environment: + <<: *COVERALLS_ENVIRONMENT + command: | + pip install coveralls==3.2.0 + python -m coveralls --finish + codechecks: docker: - <<: *DOCKERHUB_AUTH @@ -198,7 +222,15 @@ jobs: - "run": name: "Upload Coverage" environment: - COVERALLS_REPO_TOKEN: "JPf16rLB7T2yjgATIxFzTsEgMdN1UNq6o" + <<: *COVERALLS_ENVIRONMENT + # Mark the data as just one piece of many because we have more + # than one instance of this job (two on Windows now, some on other + # platforms later) which collects and reports coverage. This is + # necessary to cause Coveralls to merge multiple coverage results + # into a single report. Note the merge only happens when we + # "finish" a particular build, as identified by its "build_num" + # (aka "service_number"). + COVERALLS_PARALLEL: "true" command: | py -<> -m pip install coveralls py -<> -m coveralls From df05ed3c8f45ef880113c75ca2825e77e4deb0e4 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 8 Aug 2023 13:53:17 -0400 Subject: [PATCH 40/75] maybe ... not gonna use tox at all here? --- .circleci/config.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index f7665cc29..66e8c1553 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -198,9 +198,6 @@ jobs: steps: - "checkout" - - "run": - <<: *INSTALL_TOX - - "run": name: "Display tool versions" command: | From 72739b0606c20c5a1cc0223dd7c1415d103ba7cb Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 8 Aug 2023 13:57:17 -0400 Subject: [PATCH 41/75] try to be sure we can do the conversion --- .circleci/config.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 66e8c1553..85cdfb815 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -235,6 +235,10 @@ jobs: - "run": name: "Convert Result Log" command: | + # The Python for which we installed subunit is not necessarily on + # %PATH% so (possibly) re-install it with the default Python. + python -m pip install subunit2junitxml junitxml + Start-Process subunit2junitxml -Wait -RedirectStandardInput test-results.subunit2 -RedirectStandardOutput test-results.xml - "store_test_results": From 179f7b4bcb3f2021cfba2a0231ef0b5fc97fe6aa Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 8 Aug 2023 14:16:55 -0400 Subject: [PATCH 42/75] get the package name right ... sigh ... --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 85cdfb815..b5f081441 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -237,7 +237,7 @@ jobs: command: | # The Python for which we installed subunit is not necessarily on # %PATH% so (possibly) re-install it with the default Python. - python -m pip install subunit2junitxml junitxml + python -m pip install subunit junitxml Start-Process subunit2junitxml -Wait -RedirectStandardInput test-results.subunit2 -RedirectStandardOutput test-results.xml From fd6c7c880d9589e6b09a33630e441aa28f9b2736 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 8 Aug 2023 15:39:17 -0400 Subject: [PATCH 43/75] try to run the program a different way --- .circleci/config.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b5f081441..c9cb657b6 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -238,8 +238,7 @@ jobs: # The Python for which we installed subunit is not necessarily on # %PATH% so (possibly) re-install it with the default Python. python -m pip install subunit junitxml - - Start-Process subunit2junitxml -Wait -RedirectStandardInput test-results.subunit2 -RedirectStandardOutput test-results.xml + subunit2junitxml --output-to=test-results.xml test-results.subunit2 - "store_test_results": path: "test-results.xml" From 1a7a552e0dd773f25f982c14e9ff61e39ce00de1 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 8 Aug 2023 16:05:22 -0400 Subject: [PATCH 44/75] where's my test output file --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index c9cb657b6..b164b42f1 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -237,6 +237,7 @@ jobs: command: | # The Python for which we installed subunit is not necessarily on # %PATH% so (possibly) re-install it with the default Python. + Set-PSDebug -Trace 2 python -m pip install subunit junitxml subunit2junitxml --output-to=test-results.xml test-results.subunit2 From 9c43a99c53deac29e28c8a408469257aa64f492b Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 8 Aug 2023 17:02:34 -0400 Subject: [PATCH 45/75] maybe python-subunit is less broken --- .circleci/config.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b164b42f1..a13ce40d2 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -237,8 +237,7 @@ jobs: command: | # The Python for which we installed subunit is not necessarily on # %PATH% so (possibly) re-install it with the default Python. - Set-PSDebug -Trace 2 - python -m pip install subunit junitxml + python -m pip install python-subunit junitxml subunit2junitxml --output-to=test-results.xml test-results.subunit2 - "store_test_results": From 7a389bb3149cc6245d8373eacb38aac9976533dd Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 9 Aug 2023 09:23:43 -0400 Subject: [PATCH 46/75] it's cpython --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index a13ce40d2..a1092741e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -118,7 +118,7 @@ workflows: {} - "windows-server-2022": - name: "Windows Server 2022, Python <>" + name: "Windows Server 2022, CPython <>" matrix: parameters: pythonVersion: From d56ac6d6a2becf03d0fe3387340dfad19512d7f0 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 9 Aug 2023 09:24:37 -0400 Subject: [PATCH 47/75] note about error behavior of subunit2junitxml --- .circleci/config.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index a1092741e..12acea784 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -238,6 +238,11 @@ jobs: # The Python for which we installed subunit is not necessarily on # %PATH% so (possibly) re-install it with the default Python. python -m pip install python-subunit junitxml + + # subunit2junitxml exits with error if the result stream it is + # converting has test failures in it! So this step might fail. + # Since the step in which we actually _ran_ the tests won't fail + # even if there are test failures, this is a good thing for now. subunit2junitxml --output-to=test-results.xml test-results.subunit2 - "store_test_results": From e04340f30a2b45ddb293c01ba8852a547733303b Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 9 Aug 2023 09:28:42 -0400 Subject: [PATCH 48/75] supposedly this will work --- .circleci/config.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 12acea784..d4a2530c3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -141,8 +141,12 @@ workflows: - "finish-coverage-report": requires: - - "Windows Server 2022, Python 3.9" - - "Windows Server 2022, Python 3.11" + # Referencing the job by "alias" (as CircleCI calls the mapping + # key) instead of the value of its "name" property causes us to + # require every instance of the job from its matrix expansion. So + # this requirement is enough to require every Windows Server 2022 + # job. + - "windows-server-2022" images: <<: *IMAGES From d8df6d12d783ea6e0362cc62437b7355fa61af9a Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 9 Aug 2023 09:34:32 -0400 Subject: [PATCH 49/75] pin/upgrade coveralls --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index d4a2530c3..6e795c4b9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -166,7 +166,7 @@ jobs: environment: <<: *COVERALLS_ENVIRONMENT command: | - pip install coveralls==3.2.0 + pip install coveralls==3.3.1 python -m coveralls --finish codechecks: @@ -233,7 +233,7 @@ jobs: # (aka "service_number"). COVERALLS_PARALLEL: "true" command: | - py -<> -m pip install coveralls + py -<> -m pip install coveralls==3.3.1 py -<> -m coveralls - "run": From 7ebb3a2eadbd34cf0bd1a02cc2b2bf2e58551b2d Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 9 Aug 2023 09:34:40 -0400 Subject: [PATCH 50/75] some comments --- .circleci/config.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 6e795c4b9..1e45c5235 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -215,9 +215,18 @@ jobs: - "run": name: "Run Unit Tests" environment: + # Configure the results location for the subunitv2-file reporter + # from subunitreporter SUBUNITREPORTER_OUTPUT_PATH: "test-results.subunit2" + + # Try to get prompt output from the reporter to avoid no-output + # timeouts. PYTHONUNBUFFERED: "1" + command: | + # Run the test suite under coverage measurement using the + # parameterized version of Python, writing subunitv2-format + # results to the file given in the environment. py -<> -b -m coverage run -m twisted.trial --reporter=subunitv2-file --rterrors allmydata - "run": From 3e1f62fd7b3b6cf8d8ff226048cf36e122345f70 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 9 Aug 2023 09:36:32 -0400 Subject: [PATCH 51/75] cut down the test suite for faster testing, again --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1e45c5235..dc73c18e7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -227,7 +227,7 @@ jobs: # Run the test suite under coverage measurement using the # parameterized version of Python, writing subunitv2-format # results to the file given in the environment. - py -<> -b -m coverage run -m twisted.trial --reporter=subunitv2-file --rterrors allmydata + py -<> -b -m coverage run -m twisted.trial --reporter=subunitv2-file --rterrors allmydata.test.test_uri - "run": name: "Upload Coverage" From e27c2e97411fcaba8d0bb105c1636fe1816bd406 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 9 Aug 2023 09:45:29 -0400 Subject: [PATCH 52/75] where's the coverage? --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index dc73c18e7..8dcb4c150 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -243,6 +243,7 @@ jobs: COVERALLS_PARALLEL: "true" command: | py -<> -m pip install coveralls==3.3.1 + py -<> -m coveralls debug py -<> -m coveralls - "run": From 38d6e5d8408e59c00f88d206639da241434dd4a3 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 9 Aug 2023 09:56:43 -0400 Subject: [PATCH 53/75] merge the "parallel" coverage files before invoking coveralls --- .circleci/config.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 8dcb4c150..487182ec3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -243,7 +243,15 @@ jobs: COVERALLS_PARALLEL: "true" command: | py -<> -m pip install coveralls==3.3.1 - py -<> -m coveralls debug + + # .coveragerc sets parallel = True so we don't have a `.coverage` + # file but a `.coverage.` file (or maybe more than + # one, but probably not). coveralls can't work with these so + # merge them before invoking it. + py -<> -m coverage combine + + # Now coveralls will be able to find the data, so have it do the + # upload. py -<> -m coveralls - "run": From 18c5f090518f694a684aae3948a89df0b7ffec29 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 9 Aug 2023 09:56:57 -0400 Subject: [PATCH 54/75] upload the coverage results to circleci too --- .circleci/config.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 487182ec3..55f881e4b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -276,6 +276,9 @@ jobs: - "store_artifacts": path: "eliot.log" + - "store_artifacts": + path: ".coverage" + pyinstaller: docker: - <<: *DOCKERHUB_AUTH From 571ded8680d09ab5a22b0a326f6b9944b843878f Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 9 Aug 2023 10:18:05 -0400 Subject: [PATCH 55/75] try to get the coveralls we already have instead of installing again --- .circleci/config.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 55f881e4b..865020c1c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -258,14 +258,15 @@ jobs: name: "Convert Result Log" command: | # The Python for which we installed subunit is not necessarily on - # %PATH% so (possibly) re-install it with the default Python. - python -m pip install python-subunit junitxml + # %PATH% so put it there. + $p = py -<> -c "import sys; print(sys.prefix)" + $env:PATH = "$env:PATH;$p\Scripts" # subunit2junitxml exits with error if the result stream it is # converting has test failures in it! So this step might fail. # Since the step in which we actually _ran_ the tests won't fail # even if there are test failures, this is a good thing for now. - subunit2junitxml --output-to=test-results.xml test-results.subunit2 + subunit2junitxml.exe --output-to=test-results.xml test-results.subunit2 - "store_test_results": path: "test-results.xml" From 4c16744199b87e7ba2700219e3a56992f851820c Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 9 Aug 2023 10:24:29 -0400 Subject: [PATCH 56/75] try to settle %PATH% once and for all at the start of the job --- .circleci/config.yml | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 865020c1c..12d3536e0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -202,15 +202,26 @@ jobs: steps: - "checkout" + - "run": + name: "Fix $env:PATH" + command: | + # The Python for which we installed subunit is not necessarily on + # %PATH% so put it there. + # gets tools from packages we install. + $p = py -<> -c "import sys; print(sys.prefix)" + New-Item $Profile.CurrentUserAllHosts -Force + # $p gets "python" on PATH and $p\Scripts + Add-Content -Path $Profile.CurrentUserAllHosts -Value '$env:PATH = "$p;$p\Scripts;$env:PATH"' + - "run": name: "Display tool versions" command: | - py -<> misc/build_helpers/show-tool-versions.py + python misc/build_helpers/show-tool-versions.py - "run": name: "Install Dependencies" command: | - py -<> -m pip install .[testenv] .[test] + python -m pip install .[testenv] .[test] - "run": name: "Run Unit Tests" @@ -227,7 +238,7 @@ jobs: # Run the test suite under coverage measurement using the # parameterized version of Python, writing subunitv2-format # results to the file given in the environment. - py -<> -b -m coverage run -m twisted.trial --reporter=subunitv2-file --rterrors allmydata.test.test_uri + python -b -m coverage run -m twisted.trial --reporter=subunitv2-file --rterrors allmydata.test.test_uri - "run": name: "Upload Coverage" @@ -242,26 +253,21 @@ jobs: # (aka "service_number"). COVERALLS_PARALLEL: "true" command: | - py -<> -m pip install coveralls==3.3.1 + python -m pip install coveralls==3.3.1 # .coveragerc sets parallel = True so we don't have a `.coverage` # file but a `.coverage.` file (or maybe more than # one, but probably not). coveralls can't work with these so # merge them before invoking it. - py -<> -m coverage combine + python -m coverage combine # Now coveralls will be able to find the data, so have it do the # upload. - py -<> -m coveralls + python -m coveralls - "run": name: "Convert Result Log" command: | - # The Python for which we installed subunit is not necessarily on - # %PATH% so put it there. - $p = py -<> -c "import sys; print(sys.prefix)" - $env:PATH = "$env:PATH;$p\Scripts" - # subunit2junitxml exits with error if the result stream it is # converting has test failures in it! So this step might fail. # Since the step in which we actually _ran_ the tests won't fail From 139a329b38c9457b7875731dedc713f89a469324 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 9 Aug 2023 10:29:18 -0400 Subject: [PATCH 57/75] debug PATH setup --- .circleci/config.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 12d3536e0..9a1f8cd41 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -213,6 +213,11 @@ jobs: # $p gets "python" on PATH and $p\Scripts Add-Content -Path $Profile.CurrentUserAllHosts -Value '$env:PATH = "$p;$p\Scripts;$env:PATH"' + - "run": + name: "Reveal $env:PATH" + command: | + $env:PATH + - "run": name: "Display tool versions" command: | From 66177ae28e653d8134ae2d2da425a44b5cd5b931 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 9 Aug 2023 10:44:00 -0400 Subject: [PATCH 58/75] how about this impressive construction? previous version was constructing the value string wrong --- .circleci/config.yml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 9a1f8cd41..e530ddeb5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -207,11 +207,15 @@ jobs: command: | # The Python for which we installed subunit is not necessarily on # %PATH% so put it there. - # gets tools from packages we install. $p = py -<> -c "import sys; print(sys.prefix)" + $q = py -<> -c "import sysconfig; print(sysconfig.get_path('scripts'))" + New-Item $Profile.CurrentUserAllHosts -Force - # $p gets "python" on PATH and $p\Scripts - Add-Content -Path $Profile.CurrentUserAllHosts -Value '$env:PATH = "$p;$p\Scripts;$env:PATH"' + # $p gets "python" on PATH and $q gets tools from packages we + # install. Note we carefully construct the string so that + # $env:PATH is not substituted now but $p and $q are. ` is the + # PowerShell string escape character. + Add-Content -Path $Profile.CurrentUserAllHosts -Value "`$env:PATH = `"$p;$q;`$env:PATH`"' - "run": name: "Reveal $env:PATH" From 0995b77020116d72abe3390a387231f124a1e599 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 9 Aug 2023 10:44:17 -0400 Subject: [PATCH 59/75] try stripping the interpreter-specific prefix from our paths --- .circleci/config.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e530ddeb5..501e28b9a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -271,8 +271,10 @@ jobs: python -m coverage combine # Now coveralls will be able to find the data, so have it do the - # upload. - python -m coveralls + # upload. Also, have it strip the system config-specific prefix + # from all of the source paths. + $prefix = python -c "import sysconfig; print(sysconfig.get_path('purelib'))" + python -m coveralls --basedir $prefix - "run": name: "Convert Result Log" From f17939009466780d1b4d114cc4e886d160047f78 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 9 Aug 2023 10:52:10 -0400 Subject: [PATCH 60/75] fix quoting bug --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 501e28b9a..050871832 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -215,7 +215,7 @@ jobs: # install. Note we carefully construct the string so that # $env:PATH is not substituted now but $p and $q are. ` is the # PowerShell string escape character. - Add-Content -Path $Profile.CurrentUserAllHosts -Value "`$env:PATH = `"$p;$q;`$env:PATH`"' + Add-Content -Path $Profile.CurrentUserAllHosts -Value "`$env:PATH = `"$p;$q;`$env:PATH`" - "run": name: "Reveal $env:PATH" From e1269c836d39571aae4594be7e1cd9f419aa7fc1 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 9 Aug 2023 10:55:49 -0400 Subject: [PATCH 61/75] try again with closing quote --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 050871832..55be820a7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -215,7 +215,7 @@ jobs: # install. Note we carefully construct the string so that # $env:PATH is not substituted now but $p and $q are. ` is the # PowerShell string escape character. - Add-Content -Path $Profile.CurrentUserAllHosts -Value "`$env:PATH = `"$p;$q;`$env:PATH`" + Add-Content -Path $Profile.CurrentUserAllHosts -Value "`$env:PATH = `"$p;$q;`$env:PATH`"" - "run": name: "Reveal $env:PATH" From 89506a6f828f8da8e845b377fa4a8d4470ca076e Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 9 Aug 2023 11:03:33 -0400 Subject: [PATCH 62/75] back to the full test suite --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 55be820a7..b7f3ffd85 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -247,7 +247,7 @@ jobs: # Run the test suite under coverage measurement using the # parameterized version of Python, writing subunitv2-format # results to the file given in the environment. - python -b -m coverage run -m twisted.trial --reporter=subunitv2-file --rterrors allmydata.test.test_uri + python -b -m coverage run -m twisted.trial --reporter=subunitv2-file --rterrors allmydata - "run": name: "Upload Coverage" From e072fb60b85f23d801fed8086608bf26db4f0dd6 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 9 Aug 2023 11:20:28 -0400 Subject: [PATCH 63/75] fix the comment above PATH manipulation --- .circleci/config.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b7f3ffd85..465a97a6b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -205,8 +205,11 @@ jobs: - "run": name: "Fix $env:PATH" command: | - # The Python for which we installed subunit is not necessarily on - # %PATH% so put it there. + # The Python this job is parameterized is not necessarily the one + # at the front of $env:PATH. Modify $env:PATH so that it is so we + # can just say "python" in the rest of the steps. Also get the + # related Scripts directory so tools from packages we install are + # also available. $p = py -<> -c "import sys; print(sys.prefix)" $q = py -<> -c "import sysconfig; print(sysconfig.get_path('scripts'))" From 35d731adf02fc3705d95884f1816b61bd6ac502c Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 9 Aug 2023 11:20:38 -0400 Subject: [PATCH 64/75] remove the debug step --- .circleci/config.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 465a97a6b..dbd275a76 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -220,11 +220,6 @@ jobs: # PowerShell string escape character. Add-Content -Path $Profile.CurrentUserAllHosts -Value "`$env:PATH = `"$p;$q;`$env:PATH`"" - - "run": - name: "Reveal $env:PATH" - command: | - $env:PATH - - "run": name: "Display tool versions" command: | From ce8a6d49c7c7e03edfe7f9893a958690b7e33089 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 9 Aug 2023 11:30:39 -0400 Subject: [PATCH 65/75] Attempt to cache packages downloaded with pip for Windows jobs --- .circleci/config.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index dbd275a76..629488137 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -198,10 +198,18 @@ jobs: executor: "windows" environment: TAHOE_LAFS_HYPOTHESIS_PROFILE: "ci" + # Tell pip where its download cache lives. This must agree with the + # "save_cache" step below or caching won't really work right. + PIP_CACHE_DIR: "pip-cache" steps: - "checkout" + - "restore_cache": + keys: + - "pip-packages-v1-{{ checksum \"setup.py\" }}" + - "pip-packages-v1-" + - "run": name: "Fix $env:PATH" command: | @@ -230,6 +238,12 @@ jobs: command: | python -m pip install .[testenv] .[test] + - "save_cache": + paths: + # Make sure this agrees with PIP_CACHE_DIR in the environment. + - "pip-cache" + key: "pip-packages-v1-{{ checksum \"setup.py\" }}" + - "run": name: "Run Unit Tests" environment: From fa72ac795166091304be631dc0911c2c9356e4cd Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 9 Aug 2023 12:00:56 -0400 Subject: [PATCH 66/75] a couple more comments about the windows job steps --- .circleci/config.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 629488137..d06d9a0a6 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -197,7 +197,11 @@ jobs: executor: "windows" environment: + # Tweak Hypothesis to make its behavior more suitable for the CI + # environment. This should improve reproducibility and lessen the + # effects of variable compute resources. TAHOE_LAFS_HYPOTHESIS_PROFILE: "ci" + # Tell pip where its download cache lives. This must agree with the # "save_cache" step below or caching won't really work right. PIP_CACHE_DIR: "pip-cache" @@ -205,6 +209,8 @@ jobs: steps: - "checkout" + # If possible, restore a pip download cache to save us from having to + # download all our Python dependencies from PyPI. - "restore_cache": keys: - "pip-packages-v1-{{ checksum \"setup.py\" }}" From c5cac7b5a7e7a3c5145c3893b6cdeab77bd64854 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 9 Aug 2023 12:01:02 -0400 Subject: [PATCH 67/75] get rid of the partial cache key CircleCI docs don't clearly explain what happens after a partial cache key match and reconstructing our cache is sufficiently cheap that it's probably not worth the complexity / uncertainty. --- .circleci/config.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index d06d9a0a6..7febae61a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -214,7 +214,6 @@ jobs: - "restore_cache": keys: - "pip-packages-v1-{{ checksum \"setup.py\" }}" - - "pip-packages-v1-" - "run": name: "Fix $env:PATH" From 4db44dc1787dc0c98b4e6513728df3ee7faf2287 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 9 Aug 2023 12:34:09 -0400 Subject: [PATCH 68/75] Attempt to cache all the wheels --- .circleci/config.yml | 33 ++++++++++++++++++++++++++++++--- 1 file changed, 30 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 7febae61a..e20587f75 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -206,6 +206,11 @@ jobs: # "save_cache" step below or caching won't really work right. PIP_CACHE_DIR: "pip-cache" + # And tell pip where it can find out cached wheelhouse for fast wheel + # installation, even for projects that don't distribute wheels. This + # must also agree with the "save_cache" step below. + PIP_FIND_LINKS: "wheelhouse" + steps: - "checkout" @@ -213,7 +218,11 @@ jobs: # download all our Python dependencies from PyPI. - "restore_cache": keys: - - "pip-packages-v1-{{ checksum \"setup.py\" }}" + # The download cache and/or the wheelhouse may contain Python + # version-specific binary packages so include the Python version + # in this key, as well as the canonical source of our + # dependencies. + - "pip-packages-v1-{{ parameters.pythonVersion }}-{{ checksum \"setup.py\" }}" - "run": name: "Fix $env:PATH" @@ -239,16 +248,34 @@ jobs: python misc/build_helpers/show-tool-versions.py - "run": - name: "Install Dependencies" + # It's faster to install a wheel than a source package. If we don't + # have a cached wheelhouse then build all of the wheels and dump + # them into a directory where they can become a cached wheelhouse. + # We would have built these wheels during installation anyway so it + # doesn't cost us anything extra and saves us effort next time. + name: "(Maybe) Build Wheels" command: | - python -m pip install .[testenv] .[test] + if ((Test-Path .\wheelhouse) -and (Test-Path .\wheelhouse\*)) { + echo "Found populated wheelhouse, skipping wheel building." + } else { + python -m pip wheel --wheel-dir $env:PIP_FIND_LINKS .[testenv] .[test] + } - "save_cache": paths: # Make sure this agrees with PIP_CACHE_DIR in the environment. - "pip-cache" + - "wheelhouse" key: "pip-packages-v1-{{ checksum \"setup.py\" }}" + - "run": + name: "Install Dependencies" + environment: + # By this point we should no longer need an index. + PIP_NO_INDEX: "1" + command: | + python -m pip install .[testenv] .[test] + - "run": name: "Run Unit Tests" environment: From 65d76c2e3c471efc192ddb1a0f6f67ddec9673b0 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 9 Aug 2023 12:53:07 -0400 Subject: [PATCH 69/75] must install `wheel` to build wheels with `pip wheel` it seems --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index e20587f75..4d2091e79 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -258,6 +258,7 @@ jobs: if ((Test-Path .\wheelhouse) -and (Test-Path .\wheelhouse\*)) { echo "Found populated wheelhouse, skipping wheel building." } else { + python -m pip install wheel python -m pip wheel --wheel-dir $env:PIP_FIND_LINKS .[testenv] .[test] } From a0389e83cc2e61bdade4d581a71f7aa0e33b7d1c Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 9 Aug 2023 12:54:02 -0400 Subject: [PATCH 70/75] use the *correct* templating system for this value --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4d2091e79..9923a3fcc 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -222,7 +222,7 @@ jobs: # version-specific binary packages so include the Python version # in this key, as well as the canonical source of our # dependencies. - - "pip-packages-v1-{{ parameters.pythonVersion }}-{{ checksum \"setup.py\" }}" + - "pip-packages-v1-<< parameters.pythonVersion >>-{{ checksum \"setup.py\" }}" - "run": name: "Fix $env:PATH" From 14135ea3f0aa2d30c009ef9176170a732e28d5c9 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 9 Aug 2023 13:07:03 -0400 Subject: [PATCH 71/75] make sure the two mentions of the cache key agree --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 9923a3fcc..d892a0efd 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -222,7 +222,7 @@ jobs: # version-specific binary packages so include the Python version # in this key, as well as the canonical source of our # dependencies. - - "pip-packages-v1-<< parameters.pythonVersion >>-{{ checksum \"setup.py\" }}" + - &CACHE_KEY "pip-packages-v1-<< parameters.pythonVersion >>-{{ checksum \"setup.py\" }}" - "run": name: "Fix $env:PATH" @@ -267,7 +267,7 @@ jobs: # Make sure this agrees with PIP_CACHE_DIR in the environment. - "pip-cache" - "wheelhouse" - key: "pip-packages-v1-{{ checksum \"setup.py\" }}" + key: *CACHE_KEY - "run": name: "Install Dependencies" From a73b6d99c4564e0275c315f7c7627c92c21ea732 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 9 Aug 2023 14:10:58 -0400 Subject: [PATCH 72/75] we end up using the coveralls tool to clean up these paths and I stopped using tox so they don't look like this anymore --- .coveragerc | 2 -- 1 file changed, 2 deletions(-) diff --git a/.coveragerc b/.coveragerc index 32b803586..5b41f9ce3 100644 --- a/.coveragerc +++ b/.coveragerc @@ -23,7 +23,5 @@ source = D:/a/tahoe-lafs/tahoe-lafs/.tox/py*-coverage/Lib/site-packages/ # Although sometimes it looks like this instead. Also it looks like this on macOS. .tox/py*-coverage/lib/python*/site-packages/ -# And on the CircleCI Windows build envronment... - .tox/py*-coverage/Lib/site-packages/ # On some Linux CI jobs it looks like this /tmp/tahoe-lafs.tox/py*-coverage/lib/python*/site-packages/ From 27b97dc1d8b8f56887dc2508d91ec6239d25dafd Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 9 Aug 2023 15:01:07 -0400 Subject: [PATCH 73/75] bump it --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 86873ad53..433721d2a 100644 --- a/setup.py +++ b/setup.py @@ -413,7 +413,7 @@ setup(name="tahoe-lafs", # also set in __init__.py "pip==22.0.3", "wheel==0.37.1", "setuptools==60.9.1", - "subunitreporter==22.2.0", + "subunitreporter==23.8.0", "python-subunit==1.4.2", "junitxml==0.7", "coverage==7.2.5", From d93d6122f78f586306a129eb09d13c81b11a2b90 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 9 Aug 2023 15:01:16 -0400 Subject: [PATCH 74/75] news fragment --- newsfragments/4059.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/4059.minor diff --git a/newsfragments/4059.minor b/newsfragments/4059.minor new file mode 100644 index 000000000..e69de29bb From a95a6b88a92f50d1c33e37e1af6bf6d743d16dde Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 9 Aug 2023 17:04:26 -0400 Subject: [PATCH 75/75] note motivation for our choice of these python versions --- .circleci/config.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index d892a0efd..d327ecbc7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -121,6 +121,13 @@ workflows: name: "Windows Server 2022, CPython <>" matrix: parameters: + # Run the job for a number of CPython versions. These are the + # two versions installed on the version of the Windows VM image + # we specify (in the executor). This is handy since it means we + # don't have to do any Python installation work. We pin the + # Windows VM image so these shouldn't shuffle around beneath us + # but if we want to update that image or get different versions + # of Python, we probably have to do something here. pythonVersion: - "3.9" - "3.11"