From c37e330efd35af20cdba267ff4a82d3df4d19efb Mon Sep 17 00:00:00 2001 From: Chris Wood Date: Fri, 27 Jan 2023 10:02:59 -0500 Subject: [PATCH 01/11] Add charset_normalizer.md__mypyc to hidden imports Fixes: https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3966 Overrides: https://github.com/tahoe-lafs/tahoe-lafs/pull/1248 Ref.: https://github.com/pyinstaller/pyinstaller-hooks-contrib/issues/534 --- pyinstaller.spec | 1 + 1 file changed, 1 insertion(+) diff --git a/pyinstaller.spec b/pyinstaller.spec index eece50757..01b1ac4ac 100644 --- a/pyinstaller.spec +++ b/pyinstaller.spec @@ -36,6 +36,7 @@ hidden_imports = [ 'allmydata.stats', 'base64', 'cffi', + 'charset_normalizer.md__mypyc', 'collections', 'commands', 'Crypto', From 87dad9bd2bf96eedfd671e17fe003932358d7157 Mon Sep 17 00:00:00 2001 From: Chris Wood Date: Fri, 27 Jan 2023 10:07:50 -0500 Subject: [PATCH 02/11] Remove "charset_normalizer < 3" constraint --- setup.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/setup.py b/setup.py index f867e901d..1974145cb 100644 --- a/setup.py +++ b/setup.py @@ -147,13 +147,6 @@ install_requires = [ # for pid-file support "psutil", "filelock", - - # treq needs requests, requests needs charset_normalizer, - # charset_normalizer breaks PyInstaller - # (https://github.com/Ousret/charset_normalizer/issues/253). So work around - # this by using a lower version number. Once upstream issue is fixed, or - # requests drops charset_normalizer, this can go away. - "charset_normalizer < 3", ] setup_requires = [ From a292f52de1aa44e34e9b72ab57d4ac608b6b5da3 Mon Sep 17 00:00:00 2001 From: Chris Wood Date: Fri, 27 Jan 2023 11:47:50 -0500 Subject: [PATCH 03/11] Try debugging CI/ubuntu-20.04 integration tests.. Does restoring the "charset_normalizer < 3" pin make the tests pass? --- setup.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/setup.py b/setup.py index 1974145cb..f867e901d 100644 --- a/setup.py +++ b/setup.py @@ -147,6 +147,13 @@ install_requires = [ # for pid-file support "psutil", "filelock", + + # treq needs requests, requests needs charset_normalizer, + # charset_normalizer breaks PyInstaller + # (https://github.com/Ousret/charset_normalizer/issues/253). So work around + # this by using a lower version number. Once upstream issue is fixed, or + # requests drops charset_normalizer, this can go away. + "charset_normalizer < 3", ] setup_requires = [ From e046627d3178846bdc581ce5f38de07f52f0cdca Mon Sep 17 00:00:00 2001 From: Chris Wood Date: Fri, 27 Jan 2023 11:59:24 -0500 Subject: [PATCH 04/11] Try debugging CI/ubuntu-20.04 integration tests... Does removing the `charset_normalizer.md__mypyc` hidden import make the tests pass? --- pyinstaller.spec | 1 - 1 file changed, 1 deletion(-) diff --git a/pyinstaller.spec b/pyinstaller.spec index 01b1ac4ac..eece50757 100644 --- a/pyinstaller.spec +++ b/pyinstaller.spec @@ -36,7 +36,6 @@ hidden_imports = [ 'allmydata.stats', 'base64', 'cffi', - 'charset_normalizer.md__mypyc', 'collections', 'commands', 'Crypto', From 15c7916e0812e6baa2a931cd54b18f3382a8456e Mon Sep 17 00:00:00 2001 From: Chris Wood Date: Fri, 27 Jan 2023 12:46:30 -0500 Subject: [PATCH 05/11] Revert previous two commits (e046627, a292f52) --- pyinstaller.spec | 1 + setup.py | 7 ------- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/pyinstaller.spec b/pyinstaller.spec index eece50757..01b1ac4ac 100644 --- a/pyinstaller.spec +++ b/pyinstaller.spec @@ -36,6 +36,7 @@ hidden_imports = [ 'allmydata.stats', 'base64', 'cffi', + 'charset_normalizer.md__mypyc', 'collections', 'commands', 'Crypto', diff --git a/setup.py b/setup.py index f867e901d..1974145cb 100644 --- a/setup.py +++ b/setup.py @@ -147,13 +147,6 @@ install_requires = [ # for pid-file support "psutil", "filelock", - - # treq needs requests, requests needs charset_normalizer, - # charset_normalizer breaks PyInstaller - # (https://github.com/Ousret/charset_normalizer/issues/253). So work around - # this by using a lower version number. Once upstream issue is fixed, or - # requests drops charset_normalizer, this can go away. - "charset_normalizer < 3", ] setup_requires = [ From ff964b2310382b92c9d0392fa94705d3b64f2dec Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Sat, 28 Jan 2023 08:53:53 -0500 Subject: [PATCH 06/11] news fragment --- newsfragments/3969.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3969.minor diff --git a/newsfragments/3969.minor b/newsfragments/3969.minor new file mode 100644 index 000000000..e69de29bb From 230ce346c578c7a0969661b60f931397d4081c36 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Sat, 28 Jan 2023 08:54:00 -0500 Subject: [PATCH 07/11] circleci env var notes --- .circleci/circleci.txt | 78 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 .circleci/circleci.txt diff --git a/.circleci/circleci.txt b/.circleci/circleci.txt new file mode 100644 index 000000000..c7adf9ec1 --- /dev/null +++ b/.circleci/circleci.txt @@ -0,0 +1,78 @@ +# A master build looks like this: + +# BASH_ENV=/tmp/.bash_env-63d018969ca480003a031e62-0-build +# CI=true +# CIRCLECI=true +# CIRCLE_BRANCH=master +# CIRCLE_BUILD_NUM=76545 +# CIRCLE_BUILD_URL=https://circleci.com/gh/tahoe-lafs/tahoe-lafs/76545 +# CIRCLE_JOB=NixOS 21.11 +# CIRCLE_NODE_INDEX=0 +# CIRCLE_NODE_TOTAL=1 +# CIRCLE_PROJECT_REPONAME=tahoe-lafs +# CIRCLE_PROJECT_USERNAME=tahoe-lafs +# CIRCLE_REPOSITORY_URL=git@github.com:tahoe-lafs/tahoe-lafs.git +# CIRCLE_SHA1=ed0bda2d7456f4a2cd60870072e1fe79864a49a1 +# CIRCLE_SHELL_ENV=/tmp/.bash_env-63d018969ca480003a031e62-0-build +# CIRCLE_USERNAME=alice +# CIRCLE_WORKFLOW_ID=6d9bb71c-be3a-4659-bf27-60954180619b +# CIRCLE_WORKFLOW_JOB_ID=0793c975-7b9f-489f-909b-8349b72d2785 +# CIRCLE_WORKFLOW_WORKSPACE_ID=6d9bb71c-be3a-4659-bf27-60954180619b +# CIRCLE_WORKING_DIRECTORY=~/project + +# A build of an in-repo PR looks like this: + +# BASH_ENV=/tmp/.bash_env-63d1971a0298086d8841287e-0-build +# CI=true +# CIRCLECI=true +# CIRCLE_BRANCH=3946-less-chatty-downloads +# CIRCLE_BUILD_NUM=76612 +# CIRCLE_BUILD_URL=https://circleci.com/gh/tahoe-lafs/tahoe-lafs/76612 +# CIRCLE_JOB=NixOS 21.11 +# CIRCLE_NODE_INDEX=0 +# CIRCLE_NODE_TOTAL=1 +# CIRCLE_PROJECT_REPONAME=tahoe-lafs +# CIRCLE_PROJECT_USERNAME=tahoe-lafs +# CIRCLE_PULL_REQUEST=https://github.com/tahoe-lafs/tahoe-lafs/pull/1251 +# CIRCLE_PULL_REQUESTS=https://github.com/tahoe-lafs/tahoe-lafs/pull/1251 +# CIRCLE_REPOSITORY_URL=git@github.com:tahoe-lafs/tahoe-lafs.git +# CIRCLE_SHA1=921a2083dcefdb5f431cdac195fc9ac510605349 +# CIRCLE_SHELL_ENV=/tmp/.bash_env-63d1971a0298086d8841287e-0-build +# CIRCLE_USERNAME=bob +# CIRCLE_WORKFLOW_ID=5e32c12e-be37-4868-9fa8-6a6929fec2f1 +# CIRCLE_WORKFLOW_JOB_ID=316ca408-81b4-4c96-bbdd-644e4c3e01e5 +# CIRCLE_WORKFLOW_WORKSPACE_ID=5e32c12e-be37-4868-9fa8-6a6929fec2f1 +# CIRCLE_WORKING_DIRECTORY=~/project +# CI_PULL_REQUEST=https://github.com/tahoe-lafs/tahoe-lafs/pull/1251 + +# A build of a PR from a fork looks like this: + +# BASH_ENV=/tmp/.bash_env-63d40f7b2e89cd3de10e0db9-0-build +# CI=true +# CIRCLECI=true +# CIRCLE_BRANCH=pull/1252 +# CIRCLE_BUILD_NUM=76678 +# CIRCLE_BUILD_URL=https://circleci.com/gh/tahoe-lafs/tahoe-lafs/76678 +# CIRCLE_JOB=NixOS 21.05 +# CIRCLE_NODE_INDEX=0 +# CIRCLE_NODE_TOTAL=1 +# CIRCLE_PROJECT_REPONAME=tahoe-lafs +# CIRCLE_PROJECT_USERNAME=tahoe-lafs +# CIRCLE_PR_NUMBER=1252 +# CIRCLE_PR_REPONAME=tahoe-lafs +# CIRCLE_PR_USERNAME=carol +# CIRCLE_PULL_REQUEST=https://github.com/tahoe-lafs/tahoe-lafs/pull/1252 +# CIRCLE_PULL_REQUESTS=https://github.com/tahoe-lafs/tahoe-lafs/pull/1252 +# CIRCLE_REPOSITORY_URL=git@github.com:tahoe-lafs/tahoe-lafs.git +# CIRCLE_SHA1=15c7916e0812e6baa2a931cd54b18f3382a8456e +# CIRCLE_SHELL_ENV=/tmp/.bash_env-63d40f7b2e89cd3de10e0db9-0-build +# CIRCLE_USERNAME= +# CIRCLE_WORKFLOW_ID=19c917c8-3a38-4b20-ac10-3265259fa03e +# CIRCLE_WORKFLOW_JOB_ID=58e95215-eccf-4664-a231-1dba7fd2d323 +# CIRCLE_WORKFLOW_WORKSPACE_ID=19c917c8-3a38-4b20-ac10-3265259fa03e +# CIRCLE_WORKING_DIRECTORY=~/project +# CI_PULL_REQUEST=https://github.com/tahoe-lafs/tahoe-lafs/pull/1252 + +# A build of a PR from a fork where the owner has enabled CircleCI looks +# the same as a build of an in-repo PR, except it runs on th owner's +# CircleCI namespace. From 3d58194c3ae73ae2048df952418bbdaaf82c6156 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Sat, 28 Jan 2023 08:56:48 -0500 Subject: [PATCH 08/11] Complexify the upstream-vs-forked detection --- .circleci/lib.sh | 95 ++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 83 insertions(+), 12 deletions(-) diff --git a/.circleci/lib.sh b/.circleci/lib.sh index 7717cdb18..b5c900371 100644 --- a/.circleci/lib.sh +++ b/.circleci/lib.sh @@ -1,21 +1,20 @@ # Run a command, enabling cache writes to cachix if possible. The command is # accepted as a variable number of positional arguments (like argv). function cache_if_able() { - # The `cachix watch-exec ...` does our cache population. When it sees - # something added to the store (I guess) it pushes it to the named cache. - # - # We can only *push* to it if we have a CACHIX_AUTH_TOKEN, though. - # in-repo jobs will get this from CircleCI configuration but jobs from - # forks may not. - echo "Building PR from user/org: ${CIRCLE_PROJECT_USERNAME}" - if [ -v CACHIX_AUTH_TOKEN ]; then + # Dump some info about our build environment. + describe_build + + if is_cache_writeable; then + # If the cache is available we'll use it. This lets fork owners set + # up their own caching if they want. echo "Cachix credentials present; will attempt to write to cache." + + # The `cachix watch-exec ...` does our cache population. When it sees + # something added to the store (I guess) it pushes it to the named + # cache. cachix watch-exec "${CACHIX_NAME}" -- "$@" else - # If we're building a from a forked repository then we're allowed to - # not have the credentials (but it's also fine if the owner of the - # fork supplied their own). - if [ "${CIRCLE_PROJECT_USERNAME}" == "tahoe-lafs" ]; then + if is_cache_required; then echo "Required credentials (CACHIX_AUTH_TOKEN) are missing." return 1 else @@ -24,3 +23,75 @@ function cache_if_able() { fi fi } + +function is_cache_writeable() { + # We can only *push* to the cache if we have a CACHIX_AUTH_TOKEN. in-repo + # jobs will get this from CircleCI configuration but jobs from forks may + # not. + [ -v CACHIX_AUTH_TOKEN ] +} + +function is_cache_required() { + # If we're building in tahoe-lafs/tahoe-lafs then we must use the cache. + # If we're building anything from a fork then we're allowed to not have + # the credentials. + is_upstream +} + +# Return success if the origin of this build is the tahoe-lafs/tahoe-lafs +# repository itself (and so we expect to have cache credentials available), +# failure otherwise. +# +# See circleci.txt for notes about how this determination is made. +function is_upstream() { + # CIRCLE_PROJECT_USERNAME is set to the org the build is happening for. + # If a PR targets a fork of the repo then this is set to something other + # than "tahoe-lafs". + [ "$CIRCLE_PROJECT_USERNAME" == "tahoe-lafs" ] && + + # CIRCLE_BRANCH is set to the real branch name for in-repo PRs and + # "pull/NNNN" for pull requests from forks. + # + # CIRCLE_PULL_REQUEST is set to the full URL of the PR page which ends + # with that same "pull/NNNN" for PRs from forks. + ! endswith "/$CIRCLE_BRANCH" "$CIRCLE_PULL_REQUEST" +} + +# Return success if $2 ends with $1, failure otherwise. +function endswith() { + suffix=$1 + shift + + haystack=$1 + shift + + case "$haystack" in + *${suffix}) + return 0 + ;; + + *) + return 1 + ;; + esac +} + +function describe_build() { + echo "Building PR for user/org: ${CIRCLE_PROJECT_USERNAME}" + echo "Building branch: ${CIRCLE_BRANCH}" + if is_upstream; then + echo "Upstream build." + else + echo "Non-upstream build." + fi + if is_cache_required; then + echo "Cache is required." + else + echo "Cache not required." + fi + if is_cache_writeable; then + echo "Cache is writeable." + else + echo "Cache not writeable." + fi +} From 4ea4286a7f52429addb4dbd1eb0467659fb2b66f Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Sat, 28 Jan 2023 09:21:34 -0500 Subject: [PATCH 09/11] Use CIRCLE_PULL_REQUESTS in case there are multiple which, of course, there never are, except for during testing of this branch --- .circleci/lib.sh | 28 +++++++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/.circleci/lib.sh b/.circleci/lib.sh index b5c900371..c692b5f88 100644 --- a/.circleci/lib.sh +++ b/.circleci/lib.sh @@ -52,9 +52,31 @@ function is_upstream() { # CIRCLE_BRANCH is set to the real branch name for in-repo PRs and # "pull/NNNN" for pull requests from forks. # - # CIRCLE_PULL_REQUEST is set to the full URL of the PR page which ends - # with that same "pull/NNNN" for PRs from forks. - ! endswith "/$CIRCLE_BRANCH" "$CIRCLE_PULL_REQUEST" + # CIRCLE_PULL_REQUESTS is set to a comma-separated list of the full + # URLs of the PR pages which share an underlying branch, with one of + # them ended with that same "pull/NNNN" for PRs from forks. + ! any_element_endswith "/$CIRCLE_BRANCH" "," "$CIRCLE_PULL_REQUESTS" +} + +# Return success if splitting $3 on $2 results in an array with any element +# that ends with $1, failure otherwise. +function any_element_endswith() { + suffix=$1 + shift + + sep=$1 + shift + + haystack=$1 + shift + + IFS="${sep}" read -r -a elements <<< "$haystack" + for elem in "${elements[@]}"; do + if endswith "$suffix" "$elem"; then + return 0 + fi + done + return 1 } # Return success if $2 ends with $1, failure otherwise. From cad81c9bdda0ad5d1bfe066d0ae8fccdcb851615 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Sat, 28 Jan 2023 16:21:45 -0500 Subject: [PATCH 10/11] Twiddle the news fragment to pass codechecks --- newsfragments/3966.bugfix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/newsfragments/3966.bugfix b/newsfragments/3966.bugfix index ead94c47c..384dcf797 100644 --- a/newsfragments/3966.bugfix +++ b/newsfragments/3966.bugfix @@ -1 +1 @@ -Fix incompatibility with newer versions of the transitive charset_normalizer dependency when using PyInstaller. \ No newline at end of file +Fix incompatibility with transitive dependency charset_normalizer >= 3 when using PyInstaller. From 9553901ca191551581e9404a81ede1d8e6bbd36f Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 31 Jan 2023 14:22:57 -0500 Subject: [PATCH 11/11] Add caveats. --- benchmarks/upload_download.py | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/benchmarks/upload_download.py b/benchmarks/upload_download.py index bd1b08e7a..aa5f506bc 100644 --- a/benchmarks/upload_download.py +++ b/benchmarks/upload_download.py @@ -3,13 +3,25 @@ First attempt at benchmarking uploads and downloads. To run: -$ pytest benchmarks/upload_download.py -s -v -Wignore +$ pytest benchmarks/upload_download.py -s -v -Wignore TODO Parameterization (pytest?) -- Foolscap vs not foolscap -- Number of nodes -- Data size -- Number of needed/happy/total shares. + + - Foolscap vs not foolscap + + - Number of nodes + + - Data size + + - Number of needed/happy/total shares. + +CAVEATS: The goal here isn't a realistic benchmark, or a benchmark that will be +measured over time, or is expected to be maintainable over time. This is just +a quick and easy way to measure the speed of certain operations, compare HTTP +and Foolscap, and see the short-term impact of changes. + +Eventually this will be replaced by a real benchmark suite that can be run over +time to measure something more meaningful. """ from time import time, process_time