From 183385a94bf550b1edc51b0b8253d127a6a275fe Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 13 Sep 2019 11:04:28 -0400 Subject: [PATCH 01/21] Disable most of CI to speed up development on this branch --- .circleci/config.yml | 56 ++++++++++++++++++++++---------------------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index cefdf8369..cdf2f4e9e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -5,41 +5,41 @@ workflows: version: 2 ci: jobs: - # Platforms - - "debian-9" - - "debian-8": - requires: - - "debian-9" + # # Platforms + # - "debian-9" + # - "debian-8": + # requires: + # - "debian-9" - - "ubuntu-18.04" - - "ubuntu-16.04": - requires: - - "ubuntu-18.04" + # - "ubuntu-18.04" + # - "ubuntu-16.04": + # requires: + # - "ubuntu-18.04" - - "fedora-29" - - "fedora-28": - requires: - - "fedora-29" + # - "fedora-29" + # - "fedora-28": + # requires: + # - "fedora-29" - - "centos-7" + # - "centos-7" - - "slackware-14.2" + # - "slackware-14.2" - # Test against PyPy 2.7/7.1.1 - - "pypy2.7-7.1" + # # Test against PyPy 2.7/7.1.1 + # - "pypy2.7-7.1" - # Other assorted tasks and configurations - - "lint" - - "deprecations" - - "c-locale" - # Any locale other than C or UTF-8. - - "another-locale" + # # Other assorted tasks and configurations + # - "lint" + # - "deprecations" + # - "c-locale" + # # Any locale other than C or UTF-8. + # - "another-locale" - - "integration": - requires: - # If the unit test suite doesn't pass, don't bother running the - # integration tests. - - "debian-9" + # - "integration": + # requires: + # # If the unit test suite doesn't pass, don't bother running the + # # integration tests. + # - "debian-9" images: # Build the Docker images used by the ci jobs. This makes the ci jobs From 42a01f00466769b63cf03ec73898ecfb43e3aee8 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 13 Sep 2019 11:05:05 -0400 Subject: [PATCH 02/21] Add a CI job which runs depgraph. It can also push the results to GitHub pages. --- .circleci/config.yml | 16 ++++++++++++++++ misc/python3/depgraph.sh | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+) create mode 100644 misc/python3/depgraph.sh diff --git a/.circleci/config.yml b/.circleci/config.yml index cdf2f4e9e..f3c4877e8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -41,6 +41,10 @@ workflows: # # integration tests. # - "debian-9" + # Generate the underlying data for a visualization to aid with Python 3 + # porting. + - "build-porting-depgraph" + images: # Build the Docker images used by the ci jobs. This makes the ci jobs # faster and takes various spurious failures out of the critical path. @@ -292,6 +296,18 @@ jobs: - store_artifacts: *STORE_OTHER_ARTIFACTS - run: *SUBMIT_COVERAGE + build-porting-depgraph: + # Generate up-to-date data for the dependency graph visualizer. + docker: + - image: "python:3.7-buster" + + steps: + - "checkout" + - run: + name: "Generate dependency graph data" + command: | + ./misc/python3/depgraph.sh + build-image: &BUILD_IMAGE # This is a template for a job to build a Docker image that has as much of # the setup as we can manage already done and baked in. This cuts down on diff --git a/misc/python3/depgraph.sh b/misc/python3/depgraph.sh new file mode 100644 index 000000000..0be456b8b --- /dev/null +++ b/misc/python3/depgraph.sh @@ -0,0 +1,35 @@ +#!/bin/env bash + +set -x +set -eo pipefail + +# if [ "${CIRCLE_BRANCH}" != "master" ]; then +# echo "Declining to update dependency graph for non-master build." +# exit 0 +# fi + +git config user.name 'Build Automation' +git config user.email 'tahoe-dev@tahoe-lafs.org' + +TAHOE="${PWD}" +git clone git@github.com:tahoe-lafs/tahoe-depgraph.git +cd tahoe-depgraph + +# Generate the maybe-changed data. +python tahoe-depgraph.py "${TAHOE}" + +if git diff-index --quiet --cached HEAD; then + echo "Declining to commit without any changes." + exit 0 +fi + +# Commit everything that changed. It should be tahoe-deps.json and +# tahoe-ported.json. +git commit -am "\ +Built from ${CIRCLE_REPOSITORY_URL}@${CIRCLE_SHA1} + +tahoe-depgraph was $(git --git-dir ../.git rev-parse HEAD +" + +# Publish it on GitHub. +git push -q origin gh-pages From 47e28d2e5d20113a1ea86425256e11603d7106cc Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 13 Sep 2019 11:21:20 -0400 Subject: [PATCH 03/21] Grant the job push access to the repository with a GitHub deploy key. --- .circleci/config.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index f3c4877e8..0c03abcd8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -303,6 +303,12 @@ jobs: steps: - "checkout" + - add_ssh_keys: + fingerprints: + # Jean-Paul Calderone (CircleCI depgraph key) + # This lets us push to tahoe-lafs/tahoe-depgraph in the next step. + - "86:38:18:a7:c0:97:42:43:18:46:55:d6:21:b0:5f:d4" + - run: name: "Generate dependency graph data" command: | From ce0d5951854bb8d8a7ea752bcb21ec216b3f6606 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 13 Sep 2019 11:23:19 -0400 Subject: [PATCH 04/21] Make it executable. --- misc/python3/depgraph.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 misc/python3/depgraph.sh diff --git a/misc/python3/depgraph.sh b/misc/python3/depgraph.sh old mode 100644 new mode 100755 From dd673a7862adb99418af25b7d32447eb73e6b006 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 13 Sep 2019 11:24:10 -0400 Subject: [PATCH 05/21] Get the path to env right --- misc/python3/depgraph.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/python3/depgraph.sh b/misc/python3/depgraph.sh index 0be456b8b..de46943ad 100755 --- a/misc/python3/depgraph.sh +++ b/misc/python3/depgraph.sh @@ -1,4 +1,4 @@ -#!/bin/env bash +#!/usr/bin/env bash set -x set -eo pipefail From 1fcf5051bd1faff992119d0ee6617d419086bab6 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 13 Sep 2019 11:25:30 -0400 Subject: [PATCH 06/21] Guessed wrong - tahoe-depgraph is Python 2. --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 0c03abcd8..386be2968 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -299,7 +299,7 @@ jobs: build-porting-depgraph: # Generate up-to-date data for the dependency graph visualizer. docker: - - image: "python:3.7-buster" + - image: "python:2.7-buster" steps: - "checkout" From a3bb83221a329d90c188079200a86615331d2ab5 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 13 Sep 2019 11:27:00 -0400 Subject: [PATCH 07/21] Get a dependency --- misc/python3/depgraph.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/misc/python3/depgraph.sh b/misc/python3/depgraph.sh index de46943ad..0a537301c 100755 --- a/misc/python3/depgraph.sh +++ b/misc/python3/depgraph.sh @@ -15,6 +15,8 @@ TAHOE="${PWD}" git clone git@github.com:tahoe-lafs/tahoe-depgraph.git cd tahoe-depgraph +pip install twisted + # Generate the maybe-changed data. python tahoe-depgraph.py "${TAHOE}" From 4dda91df0e2e4a0caf08a8c9430c9ddde24e5984 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 13 Sep 2019 11:38:30 -0400 Subject: [PATCH 08/21] Add a list of modules already ported to Python 3. --- misc/python3/ported-modules.txt | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 misc/python3/ported-modules.txt diff --git a/misc/python3/ported-modules.txt b/misc/python3/ported-modules.txt new file mode 100644 index 000000000..e69de29bb From fae2fa4bef77d14a3d9795da9b3d8b7614edd97f Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 13 Sep 2019 12:25:03 -0400 Subject: [PATCH 09/21] This looks Python 3 compatible to me. --- misc/python3/ported-modules.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/misc/python3/ported-modules.txt b/misc/python3/ported-modules.txt index e69de29bb..cd150cbfb 100644 --- a/misc/python3/ported-modules.txt +++ b/misc/python3/ported-modules.txt @@ -0,0 +1 @@ +allmydata.util.namespace From 5fd44f1914a88c4f7e8398f1c0a40e460f3cf1fa Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 13 Sep 2019 12:28:55 -0400 Subject: [PATCH 10/21] Start from the right branch --- misc/python3/depgraph.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/python3/depgraph.sh b/misc/python3/depgraph.sh index 0a537301c..f4c5b3718 100755 --- a/misc/python3/depgraph.sh +++ b/misc/python3/depgraph.sh @@ -12,7 +12,7 @@ git config user.name 'Build Automation' git config user.email 'tahoe-dev@tahoe-lafs.org' TAHOE="${PWD}" -git clone git@github.com:tahoe-lafs/tahoe-depgraph.git +git clone -b gh-pages git@github.com:tahoe-lafs/tahoe-depgraph.git cd tahoe-depgraph pip install twisted From 0daefad3a95effcf57c95a8de0ad39d5823880c4 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 13 Sep 2019 12:48:42 -0400 Subject: [PATCH 11/21] It's okay. Ask the filesystem. --- misc/python3/depgraph.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/python3/depgraph.sh b/misc/python3/depgraph.sh index f4c5b3718..16f7343ae 100755 --- a/misc/python3/depgraph.sh +++ b/misc/python3/depgraph.sh @@ -20,7 +20,7 @@ pip install twisted # Generate the maybe-changed data. python tahoe-depgraph.py "${TAHOE}" -if git diff-index --quiet --cached HEAD; then +if git diff-index --quiet HEAD; then echo "Declining to commit without any changes." exit 0 fi From bfa2cb8816af3ee0e36668a7fd93c3ce0dd93432 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 13 Sep 2019 12:50:05 -0400 Subject: [PATCH 12/21] Close paren needed --- misc/python3/depgraph.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/python3/depgraph.sh b/misc/python3/depgraph.sh index 16f7343ae..b9d43fab1 100755 --- a/misc/python3/depgraph.sh +++ b/misc/python3/depgraph.sh @@ -30,7 +30,7 @@ fi git commit -am "\ Built from ${CIRCLE_REPOSITORY_URL}@${CIRCLE_SHA1} -tahoe-depgraph was $(git --git-dir ../.git rev-parse HEAD +tahoe-depgraph was $(git --git-dir ../.git rev-parse HEAD) " # Publish it on GitHub. From 227451732fbd9e77d9d9778c95840729c1b0abdd Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 13 Sep 2019 12:51:06 -0400 Subject: [PATCH 13/21] Get the git config into the right repository --- misc/python3/depgraph.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/misc/python3/depgraph.sh b/misc/python3/depgraph.sh index b9d43fab1..5f0ead395 100755 --- a/misc/python3/depgraph.sh +++ b/misc/python3/depgraph.sh @@ -8,9 +8,6 @@ set -eo pipefail # exit 0 # fi -git config user.name 'Build Automation' -git config user.email 'tahoe-dev@tahoe-lafs.org' - TAHOE="${PWD}" git clone -b gh-pages git@github.com:tahoe-lafs/tahoe-depgraph.git cd tahoe-depgraph @@ -27,6 +24,9 @@ fi # Commit everything that changed. It should be tahoe-deps.json and # tahoe-ported.json. +git config user.name 'Build Automation' +git config user.email 'tahoe-dev@tahoe-lafs.org' + git commit -am "\ Built from ${CIRCLE_REPOSITORY_URL}@${CIRCLE_SHA1} From 00394de6768e5d1abae1be3a4f50bdbebd7f5b67 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 13 Sep 2019 12:52:11 -0400 Subject: [PATCH 14/21] This should report the tahoe-depgraph revision. --- misc/python3/depgraph.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/python3/depgraph.sh b/misc/python3/depgraph.sh index 5f0ead395..fd08cb47b 100755 --- a/misc/python3/depgraph.sh +++ b/misc/python3/depgraph.sh @@ -30,7 +30,7 @@ git config user.email 'tahoe-dev@tahoe-lafs.org' git commit -am "\ Built from ${CIRCLE_REPOSITORY_URL}@${CIRCLE_SHA1} -tahoe-depgraph was $(git --git-dir ../.git rev-parse HEAD) +tahoe-depgraph was $(git rev-parse HEAD) " # Publish it on GitHub. From b462d294a56ae94c5724e279d7d96d1e09e9e2a7 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 13 Sep 2019 13:00:35 -0400 Subject: [PATCH 15/21] Try it in a working environment ... ? --- .circleci/config.yml | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 386be2968..109658bd6 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -296,19 +296,27 @@ jobs: - store_artifacts: *STORE_OTHER_ARTIFACTS - run: *SUBMIT_COVERAGE + # Generate up-to-date data for the dependency graph visualizer. build-porting-depgraph: - # Generate up-to-date data for the dependency graph visualizer. - docker: - - image: "python:2.7-buster" + # Get a system in which we can easily install Tahoe-LAFS and all its + # dependencies. The dependency graph analyzer works by executing the code. + # It's Python, what do you expect? + <<: *DEBIAN steps: - "checkout" + - add_ssh_keys: fingerprints: # Jean-Paul Calderone (CircleCI depgraph key) # This lets us push to tahoe-lafs/tahoe-depgraph in the next step. - "86:38:18:a7:c0:97:42:43:18:46:55:d6:21:b0:5f:d4" + - run: + name: "Setup Python Environment" + command: | + /tmp/venv/bin/pip install -e /tmp/project + - run: name: "Generate dependency graph data" command: | From e51de525420db2ede46386ae8853f288b5088684 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 13 Sep 2019 13:01:34 -0400 Subject: [PATCH 16/21] should be done already --- misc/python3/depgraph.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/misc/python3/depgraph.sh b/misc/python3/depgraph.sh index fd08cb47b..5d5c649dc 100755 --- a/misc/python3/depgraph.sh +++ b/misc/python3/depgraph.sh @@ -12,8 +12,6 @@ TAHOE="${PWD}" git clone -b gh-pages git@github.com:tahoe-lafs/tahoe-depgraph.git cd tahoe-depgraph -pip install twisted - # Generate the maybe-changed data. python tahoe-depgraph.py "${TAHOE}" From 151b7768c60aac76a311e975a7deeee57da3412c Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 13 Sep 2019 13:02:55 -0400 Subject: [PATCH 17/21] make the environment available to anything in the script --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 109658bd6..f2aa8c55b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -320,6 +320,7 @@ jobs: - run: name: "Generate dependency graph data" command: | + . /tmp/venv/bin/activate ./misc/python3/depgraph.sh build-image: &BUILD_IMAGE From 9c5dce3cc8e17969eaf062d321bbfcb378bea6b8 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 13 Sep 2019 13:09:36 -0400 Subject: [PATCH 18/21] Largely done now so re-enable the rest of CI --- .circleci/config.yml | 56 ++++++++++++++++++++++---------------------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index f2aa8c55b..8a3abd994 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -5,41 +5,41 @@ workflows: version: 2 ci: jobs: - # # Platforms - # - "debian-9" - # - "debian-8": - # requires: - # - "debian-9" + # Platforms + - "debian-9" + - "debian-8": + requires: + - "debian-9" - # - "ubuntu-18.04" - # - "ubuntu-16.04": - # requires: - # - "ubuntu-18.04" + - "ubuntu-18.04" + - "ubuntu-16.04": + requires: + - "ubuntu-18.04" - # - "fedora-29" - # - "fedora-28": - # requires: - # - "fedora-29" + - "fedora-29" + - "fedora-28": + requires: + - "fedora-29" - # - "centos-7" + - "centos-7" - # - "slackware-14.2" + - "slackware-14.2" - # # Test against PyPy 2.7/7.1.1 - # - "pypy2.7-7.1" + # Test against PyPy 2.7/7.1.1 + - "pypy2.7-7.1" - # # Other assorted tasks and configurations - # - "lint" - # - "deprecations" - # - "c-locale" - # # Any locale other than C or UTF-8. - # - "another-locale" + # Other assorted tasks and configurations + - "lint" + - "deprecations" + - "c-locale" + # Any locale other than C or UTF-8. + - "another-locale" - # - "integration": - # requires: - # # If the unit test suite doesn't pass, don't bother running the - # # integration tests. - # - "debian-9" + - "integration": + requires: + # If the unit test suite doesn't pass, don't bother running the + # integration tests. + - "debian-9" # Generate the underlying data for a visualization to aid with Python 3 # porting. From 32492ec28a69019305cadec61f55246f7c5ab244 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 13 Sep 2019 13:10:02 -0400 Subject: [PATCH 19/21] Only run for master from now on --- misc/python3/depgraph.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/misc/python3/depgraph.sh b/misc/python3/depgraph.sh index 5d5c649dc..0d30ef5c3 100755 --- a/misc/python3/depgraph.sh +++ b/misc/python3/depgraph.sh @@ -3,10 +3,10 @@ set -x set -eo pipefail -# if [ "${CIRCLE_BRANCH}" != "master" ]; then -# echo "Declining to update dependency graph for non-master build." -# exit 0 -# fi +if [ "${CIRCLE_BRANCH}" != "master" ]; then + echo "Declining to update dependency graph for non-master build." + exit 0 +fi TAHOE="${PWD}" git clone -b gh-pages git@github.com:tahoe-lafs/tahoe-depgraph.git From 7bfef2d6e035b9b9a679e6c95500adb25c74bbf6 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 16 Sep 2019 09:21:05 -0400 Subject: [PATCH 20/21] Tell git about the changed files specifically. --- misc/python3/depgraph.sh | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/misc/python3/depgraph.sh b/misc/python3/depgraph.sh index 0d30ef5c3..6e5457537 100755 --- a/misc/python3/depgraph.sh +++ b/misc/python3/depgraph.sh @@ -20,12 +20,11 @@ if git diff-index --quiet HEAD; then exit 0 fi -# Commit everything that changed. It should be tahoe-deps.json and -# tahoe-ported.json. git config user.name 'Build Automation' git config user.email 'tahoe-dev@tahoe-lafs.org' -git commit -am "\ +git add tahoe-deps.json tahoe-ported.json. +git commit -m "\ Built from ${CIRCLE_REPOSITORY_URL}@${CIRCLE_SHA1} tahoe-depgraph was $(git rev-parse HEAD) From 40e29b7cbf2cf592795b6abba5f435c05b14388e Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 16 Sep 2019 09:21:18 -0400 Subject: [PATCH 21/21] news fragment --- newsfragments/3152.other | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/3152.other diff --git a/newsfragments/3152.other b/newsfragments/3152.other new file mode 100644 index 000000000..31ec507ef --- /dev/null +++ b/newsfragments/3152.other @@ -0,0 +1 @@ +Progress towards Python 3 compatibility is now visible at .