Merge pull request #956 from LeastAuthority/3385.coveralls-for-coverage-reports

Use coveralls for coverage reports

Fixes: ticket:3385

.circleci/config.yml

@@ -208,7 +208,7 @@ jobs:
       # filenames and argv).
       LANG: "en_US.UTF-8"
       # Select a tox environment to run for this job.
-      TAHOE_LAFS_TOX_ENVIRONMENT: "py27-coverage"
+      TAHOE_LAFS_TOX_ENVIRONMENT: "py27"
       # Additional arguments to pass to tox.
       TAHOE_LAFS_TOX_ARGS: ""
       # The path in which test artifacts will be placed.
@@ -218,7 +218,7 @@ jobs:
       WHEELHOUSE_PATH: &WHEELHOUSE_PATH "/tmp/wheelhouse"
       PIP_FIND_LINKS: "file:///tmp/wheelhouse"
       # Upload the coverage report.
-      UPLOAD_COVERAGE: "yes"
+      UPLOAD_COVERAGE: ""
 
     # pip cannot install packages if the working directory is not readable.
     # We want to run a lot of steps as nobody instead of as root.
@@ -371,7 +371,7 @@ jobs:
       # this reporter on Python 3.  So drop that and just specify the
       # reporter.
       TAHOE_LAFS_TRIAL_ARGS: "--reporter=subunitv2-file"
-      TAHOE_LAFS_TOX_ENVIRONMENT: "py36-coverage"
+      TAHOE_LAFS_TOX_ENVIRONMENT: "py36"
 
 
   ubuntu-20-04:

.codecov.yml

@@ -1,48 +0,0 @@
-# Override defaults for codecov.io checks.
-#
-# Documentation is at https://docs.codecov.io/docs/codecov-yaml;
-# reference is at https://docs.codecov.io/docs/codecovyml-reference.
-#
-# To validate this file, use:
-#
-#   curl --data-binary @.codecov.yml https://codecov.io/validate
-#
-# Codecov's defaults seem to leave red marks in GitHub CI checks in a
-# rather arbitrary manner, probably because of non-determinism in
-# coverage (see https://tahoe-lafs.org/trac/tahoe-lafs/ticket/2891)
-# and maybe because computers are bad with floating point numbers.
-
-# Allow coverage percentage a precision of zero decimals, and round to
-# the nearest number (for example, 89.957 to to 90; 89.497 to 89%).
-# Coverage above 90% is good, below 80% is bad.
-coverage:
-  round: nearest
-  range: 80..90
-  precision: 0
-
-  # Aim for a target test coverage of 90% in codecov/project check (do
-  # not allow project coverage to drop below that), and allow
-  # codecov/patch a threshold of 1% (allow coverage in changes to drop
-  # by that much, and no less).  That should be good enough for us.
-  status:
-    project:
-      default:
-        target: 90%
-        threshold: 1%
-    patch:
-      default:
-        threshold: 1%
-
-
-codecov:
-  # This is a public repository so supposedly we don't "need" to use an upload
-  # token.  However, using one makes sure that CI jobs running against forked
-  # repositories have coverage uploaded to the right place in codecov so
-  # their reports aren't incomplete.
-  token: "abf679b6-e2e6-4b33-b7b5-6cfbd41ee691"
-
-  notify:
-    # The reference documentation suggests that this is the default setting:
-    # https://docs.codecov.io/docs/codecovyml-reference#codecovnotifywait_for_ci
-    # However observation suggests otherwise.
-    wait_for_ci: true

.github/workflows/ci.yml

@@ -79,11 +79,110 @@ jobs:
           name: eliot.log
           path: eliot.log
 
-      - name: Upload coverage report
-        uses: codecov/codecov-action@v1
-        with:
-          token: abf679b6-e2e6-4b33-b7b5-6cfbd41ee691
-          file: coverage.xml
+      # Upload this job's coverage data to Coveralls.  While there is a GitHub
+      # Action for this, as of Jan 2021 it does not support Python coverage
+      # files - only lcov files.  Therefore, we use coveralls-python, the
+      # coveralls.io-supplied Python reporter, for this.
+      - name: "Report Coverage to Coveralls"
+        run: |
+          pip install coveralls
+          python -m coveralls
+        env:
+          # Some magic value required for some magic reason.
+          GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
+          # Help coveralls identify our project.
+          COVERALLS_REPO_TOKEN: "JPf16rLB7T2yjgATIxFzTsEgMdN1UNq6o"
+          # Every source of coverage reports needs a unique "flag name".
+          # Construct one by smashing a few variables from the matrix together
+          # here.
+          COVERALLS_FLAG_NAME: "run-${{ matrix.os }}-${{ matrix.python-version }}"
+          # Mark the data as just one piece of many because we have more than
+          # one instance of this job (Windows, macOS) which collects and
+          # reports coverage.  This is necessary to cause Coveralls to merge
+          # multiple coverage results into a single report.  Note the merge
+          # only happens when we "finish" a particular build, as identified by
+          # its "build_num" (aka "service_number").
+          COVERALLS_PARALLEL: true
+
+  # Tell Coveralls that we're done reporting coverage data.  Since we're using
+  # the "parallel" mode where more than one coverage data file is merged into
+  # a single report, we have to tell Coveralls when we've uploaded all of the
+  # data files.  This does it.  We make sure it runs last by making it depend
+  # on *all* of the coverage-collecting jobs.
+  finish-coverage-report:
+    # There happens to just be one coverage-collecting job at the moment.  If
+    # the coverage reports are broken and someone added more
+    # coverage-collecting jobs to this workflow but didn't update this, that's
+    # why.
+    needs:
+      - "coverage"
+    runs-on: "ubuntu-latest"
+    steps:
+      - name: "Check out Tahoe-LAFS sources"
+        uses: "actions/checkout@v2"
+
+      - name: "Finish Coveralls Reporting"
+        run: |
+          # coveralls-python does have a `--finish` option but it doesn't seem
+          # to work, at least for us.
+          # https://github.com/coveralls-clients/coveralls-python/issues/248
+          #
+          # But all it does is this simple POST so we can just send it
+          # ourselves.  The only hard part is guessing what the POST
+          # parameters mean.  And I've done that for you already.
+          #
+          # Since the build is done I'm going to guess that "done" is a fine
+          # value for status.
+          #
+          # That leaves "build_num".  The coveralls documentation gives some
+          # hints about it.  It suggests using $CIRCLE_WORKFLOW_ID if your job
+          # is on CircleCI.  CircleCI documentation says this about
+          # CIRCLE_WORKFLOW_ID:
+          #
+          # Observation of the coveralls.io web interface, logs from the
+          # coveralls command in action, and experimentation suggests the
+          # value for PRs is something more like:
+          #
+          #    <GIT MERGE COMMIT HASH>-PR-<PR NUM>
+          #
+          # For branches, it's just the git branch tip hash.
+
+          # For pull requests, refs/pull/<PR NUM>/merge was just checked out
+          # by so HEAD will refer to the right revision.  For branches, HEAD
+          # is also the tip of the branch.
+          REV=$(git rev-parse HEAD)
+
+          # We can get the PR number from the "context".
+          #
+          # https://docs.github.com/en/free-pro-team@latest/developers/webhooks-and-events/webhook-events-and-payloads#pull_request
+          #
+          # (via <https://github.community/t/github-ref-is-inconsistent/17728/3>).
+          #
+          # If this is a pull request, `github.event` is a `pull_request`
+          # structure which has `number` right in it.
+          #
+          # If this is a push, `github.event` is a `push` instead but we only
+          # need the revision to construct the build_num.
+
+          PR=${{ github.event.number }}
+
+          if [ "${PR}" = "" ]; then
+              BUILD_NUM=$REV
+          else
+              BUILD_NUM=$REV-PR-$PR
+          fi
+          REPO_NAME=$GITHUB_REPOSITORY
+
+          curl \
+            -k \
+            https://coveralls.io/webhook?repo_token=$COVERALLS_REPO_TOKEN \
+            -d \
+            "payload[build_num]=$BUILD_NUM&payload[status]=done&payload[repo_name]=$REPO_NAME"
+        env:
+          # Some magic value required for some magic reason.
+          GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
+          # Help coveralls identify our project.
+          COVERALLS_REPO_TOKEN: "JPf16rLB7T2yjgATIxFzTsEgMdN1UNq6o"
 
   integration:
     runs-on: ${{ matrix.os }}

README.rst

@@ -6,7 +6,7 @@ Free and Open decentralized data store
 
 `Tahoe-LAFS <https://www.tahoe-lafs.org>`__ (Tahoe Least-Authority File Store) is the first free software / open-source storage technology that distributes your data across multiple servers. Even if some servers fail or are taken over by an attacker, the entire file store continues to function correctly, preserving your privacy and security.
 
-|Contributor Covenant|  |readthedocs|  |travis|  |circleci|  |codecov|
+|Contributor Covenant|  |readthedocs|  |travis|  |circleci|  |coveralls|
 
 
 Table of contents
@@ -125,9 +125,9 @@ See `TGPPL.PDF <https://tahoe-lafs.org/~zooko/tgppl.pdf>`__ for why the TGPPL ex
 .. |circleci| image:: https://circleci.com/gh/tahoe-lafs/tahoe-lafs.svg?style=svg
     :target: https://circleci.com/gh/tahoe-lafs/tahoe-lafs
 
-.. |codecov| image:: https://codecov.io/github/tahoe-lafs/tahoe-lafs/coverage.svg?branch=master
-    :alt: test coverage percentage
-    :target: https://codecov.io/github/tahoe-lafs/tahoe-lafs?branch=master
+.. |coveralls| image:: https://coveralls.io/repos/github/tahoe-lafs/tahoe-lafs/badge.svg
+    :alt: code coverage
+    :target: https://coveralls.io/github/tahoe-lafs/tahoe-lafs
 
 .. |Contributor Covenant| image:: https://img.shields.io/badge/Contributor%20Covenant-v2.0%20adopted-ff69b4.svg
     :alt: code of conduct