Merge pull request #796 from tahoe-lafs/3355.python3-coverage-reports

Collect and report Python 3 test suite coverage

Fixes: ticket:3355

.appveyor.yml

@@ -1,95 +0,0 @@
-# adapted from https://packaging.python.org/en/latest/appveyor/
-
-environment:
-
-  matrix:
-
-    # For Python versions available on Appveyor, see
-    # http://www.appveyor.com/docs/installed-software#python
-    - PYTHON: "C:\\Python27"
-    - PYTHON: "C:\\Python27-x64"
-    #  DISTUTILS_USE_SDK: "1"
-    #  TOX_TESTENV_PASSENV: "DISTUTILS_USE_SDK INCLUDE LIB"
-
-install:
-  - |
-    %PYTHON%\python.exe -m pip install -U pip
-    %PYTHON%\python.exe -m pip install wheel tox==3.9.0 virtualenv
-
-# note:
-# %PYTHON% has: python.exe
-# %PYTHON%\Scripts has: pip.exe, tox.exe (and others installed by bare pip)
-
-# We have a custom "build" system.  We don't need MSBuild or whatever.
-build: off
-
-# Do not build feature branch with open pull requests.  This is documented but
-# it's not clear it does anything.
-skip_branch_with_pr: true
-
-# This, perhaps, is effective.
-branches:
-  # whitelist
-  only:
-    - 'master'
-
-skip_commits:
- files:
-   # The Windows builds are unaffected by news fragments.
-   - 'newsfragments/*'
-   # Also, all this build junk.
-   - '.circleci/*'
-   - '.lgtm.yml'
-   - '.travis.yml'
-
-# we run from C:\projects\tahoe-lafs
-
-test_script:
-  # Put your test command here.
-  # Note that you must use the environment variable %PYTHON% to refer to
-  # the interpreter you're using - Appveyor does not do anything special
-  # to put the Python version you want to use on PATH.
-  - |
-    %PYTHON%\Scripts\tox.exe -e coverage
-    %PYTHON%\Scripts\tox.exe -e pyinstaller
-  # To verify that the resultant PyInstaller-generated binary executes
-  # cleanly (i.e., that it terminates with an exit code of 0 and isn't
-  # failing due to import/packaging-related errors, etc.).
-  - dist\Tahoe-LAFS\tahoe.exe --version
-
-after_test:
-  # This builds the main tahoe wheel, and wheels for all dependencies.
-  # Again, you only need build.cmd if you're building C extensions for
-  # 64-bit Python 3.3/3.4. And you need to use %PYTHON% to get the correct
-  # interpreter. If _trial_temp still exists, the "pip wheel" fails on
-  # _trial_temp\local_dir (not sure why).
-  - |
-    copy _trial_temp\test.log trial_test_log.txt
-    rd /s /q _trial_temp
-    %PYTHON%\python.exe setup.py bdist_wheel
-    %PYTHON%\python.exe -m pip wheel -w dist .
-  - |
-    %PYTHON%\python.exe -m pip install codecov "coverage ~= 4.5"
-    %PYTHON%\python.exe -m coverage xml -o coverage.xml -i
-    %PYTHON%\python.exe -m codecov -X search -X gcov -f coverage.xml
-
-artifacts:
-  # bdist_wheel puts your built wheel in the dist directory
-  # "pip wheel -w dist ." puts all the dependency wheels there too
-  # this gives us a zipfile with everything
-  - path: 'dist\*'
-  - path: trial_test_log.txt
-    name: Trial test.log
-  - path: eliot.log
-    name: Eliot test log
-
-on_failure:
-  # Artifacts are not normally uploaded when the job fails.  To get the test
-  # logs, we have to push them ourselves.
-  - ps: Push-AppveyorArtifact _trial_temp\test.log -Filename trial.log
-  - ps: Push-AppveyorArtifact eliot.log -Filename eliot.log
-
-#on_success:
-#  You can use this step to upload your artifacts to a public website.
-#  See Appveyor's documentation for more details. Or you can simply
-#  access your wheels from the Appveyor "artifacts" tab for your build.

.circleci/config.yml

@@ -285,7 +285,7 @@ jobs:
       # this reporter on Python 3.  So drop that and just specify the
       # reporter.
       TAHOE_LAFS_TRIAL_ARGS: "--reporter=subunitv2-file"
-      TAHOE_LAFS_TOX_ENVIRONMENT: "py36"
+      TAHOE_LAFS_TOX_ENVIRONMENT: "py36-coverage"
 
 
   ubuntu-20.04:

.github/workflows/ci.yml

@@ -49,8 +49,8 @@ jobs:
       - name: Display tool versions
         run: python misc/build_helpers/show-tool-versions.py
 
-      - name: Run "tox -e coverage"
-        run: tox -e coverage
+      - name: Run "tox -e py27-coverage"
+        run: tox -e py27-coverage
 
       - name: Upload eliot.log in case of failure
         uses: actions/upload-artifact@v1

docs/how_to_make_a_tahoe-lafs_release.org

@@ -36,7 +36,7 @@ people are Release Maintainers:
   - [ ] documentation is ready (see above)
   - [ ] (Release Maintainer): git tag -s -u 0xE34E62D06D0E69CFCA4179FFBDE0D31D68666A7A -m "release Tahoe-LAFS-X.Y.Z" tahoe-lafs-X.Y.Z
   - [ ] build code locally:
-     tox -e py27,codechecks,coverage,deprecations,docs,integration,upcoming-deprecations
+     tox -e py27,codechecks,deprecations,docs,integration,upcoming-deprecations
   - [ ] created tarballs (they'll be in dist/ for later comparison)
      tox -e tarballs
   - [ ] release version is reporting itself as intended version

newsfragments/3355.other

@@ -0,0 +1 @@
+The "coverage" tox environment has been replaced by the "py27-coverage" and "py36-coverage" environments.

tox.ini

@@ -44,13 +44,32 @@ usedevelop = False
 # We use extras=test to get things like "mock" that are required for our unit
 # tests.
 extras = test
-commands =
-         trial {env:TAHOE_LAFS_TRIAL_ARGS:--rterrors} {posargs:allmydata}
-         tahoe --version
 
-[testenv:py36]
+setenv =
+       # Define TEST_SUITE in the environment as an aid to constructing the
+       # correct test command below.
+       !py36: TEST_SUITE = allmydata
+       py36: TEST_SUITE = allmydata.test.python3_tests
+
 commands =
-         trial {env:TAHOE_LAFS_TRIAL_ARGS:--rterrors} {posargs:allmydata.test.python3_tests}
+         # As an aid to debugging, dump all of the Python packages and their
+         # versions that are installed in the test environment.  This is
+         # particularly useful to get from CI runs - though hopefully the
+         # version pinning we do limits the variability of this output
+         pip freeze
+
+         # The tahoe script isn't sufficiently ported for this to succeed on
+         # Python 3.x yet.
+         !py36: tahoe --version
+
+         !coverage: trial {env:TAHOE_LAFS_TRIAL_ARGS:--rterrors} {posargs:{env:TEST_SUITE}}
+
+         # measuring coverage is somewhat slower than not measuring coverage
+         # so only do it on request.
+         coverage: coverage run -m twisted.trial {env:TAHOE_LAFS_TRIAL_ARGS:--rterrors --reporter=timing} {posargs:{env:TEST_SUITE}}
+         coverage: coverage combine
+         coverage: coverage xml
+
 
 [testenv:integration]
 setenv =
@@ -61,19 +80,6 @@ commands =
          coverage combine
          coverage report
 
-[testenv:coverage]
-# coverage (with --branch) takes about 65% longer to run
-commands =
-	 # As an aid to debugging, dump all of the Python packages and their
-	 # versions that are installed in the test environment.  This is
-	 # particularly useful to get from CI runs - though hopefully the
-	 # version pinning we do limits the variability of this output
-	 # somewhat.
-         pip freeze
-         tahoe --version
-         coverage run --branch -m twisted.trial {env:TAHOE_LAFS_TRIAL_ARGS:--rterrors --reporter=timing} {posargs:allmydata}
-         coverage combine
-         coverage xml
 
 [testenv:codechecks]
 # On macOS, git inside of towncrier needs $HOME.