Merge pull request #658 from tahoe-lafs/3251.python3-tahoe-depgraph

Generate module dependency graph data (wrt Python 3 porting) on CI.

.circleci/config.yml

@@ -41,6 +41,10 @@ workflows:
             # integration tests.
             - "debian-9"
 
+      # Generate the underlying data for a visualization to aid with Python 3
+      # porting.
+      - "build-porting-depgraph"
+
   images:
     # Build the Docker images used by the ci jobs.  This makes the ci jobs
     # faster and takes various spurious failures out of the critical path.
@@ -292,6 +296,33 @@ jobs:
       - store_artifacts: *STORE_OTHER_ARTIFACTS
       - run: *SUBMIT_COVERAGE
 
+  # Generate up-to-date data for the dependency graph visualizer.
+  build-porting-depgraph:
+    # Get a system in which we can easily install Tahoe-LAFS and all its
+    # dependencies.  The dependency graph analyzer works by executing the code.
+    # It's Python, what do you expect?
+    <<: *DEBIAN
+
+    steps:
+      - "checkout"
+
+      - add_ssh_keys:
+          fingerprints:
+            # Jean-Paul Calderone <exarkun@twistedmatrix.com> (CircleCI depgraph key)
+            # This lets us push to tahoe-lafs/tahoe-depgraph in the next step.
+            - "86:38:18:a7:c0:97:42:43:18:46:55:d6:21:b0:5f:d4"
+
+      - run:
+          name: "Setup Python Environment"
+          command: |
+            /tmp/venv/bin/pip install -e /tmp/project
+
+      - run:
+          name: "Generate dependency graph data"
+          command: |
+            . /tmp/venv/bin/activate
+            ./misc/python3/depgraph.sh
+
   build-image: &BUILD_IMAGE
     # This is a template for a job to build a Docker image that has as much of
     # the setup as we can manage already done and baked in.  This cuts down on

misc/python3/depgraph.sh

@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+
+set -x
+set -eo pipefail
+
+if [ "${CIRCLE_BRANCH}" != "master" ]; then
+    echo "Declining to update dependency graph for non-master build."
+    exit 0
+fi
+
+TAHOE="${PWD}"
+git clone -b gh-pages git@github.com:tahoe-lafs/tahoe-depgraph.git
+cd tahoe-depgraph
+
+# Generate the maybe-changed data.
+python tahoe-depgraph.py "${TAHOE}"
+
+if git diff-index --quiet HEAD; then
+  echo "Declining to commit without any changes."
+  exit 0
+fi
+
+git config user.name 'Build Automation'
+git config user.email 'tahoe-dev@tahoe-lafs.org'
+
+git add tahoe-deps.json tahoe-ported.json.
+git commit -m "\
+Built from ${CIRCLE_REPOSITORY_URL}@${CIRCLE_SHA1}
+
+tahoe-depgraph was $(git rev-parse HEAD)
+"
+
+# Publish it on GitHub.
+git push -q origin gh-pages

misc/python3/ported-modules.txt

@@ -0,0 +1 @@
+allmydata.util.namespace

newsfragments/3152.other

@@ -0,0 +1 @@
+Progress towards Python 3 compatibility is now visible at <https://tahoe-lafs.github.io/tahoe-depgraph/>.