Merge branch 'master' into 3448.convert-only-unicode-to-str

This commit is contained in:
Ross Patterson 2020-09-30 11:50:42 -07:00
commit fb87daad0b
19 changed files with 236 additions and 43 deletions

View File

@ -10,3 +10,7 @@ omit =
*/allmydata/_version.py
parallel = True
branch = True
[report]
show_missing = True
skip_covered = True

1
.gitignore vendored
View File

@ -45,6 +45,7 @@ zope.interface-*.egg
/.tox/
/docs/_build/
/coverage.xml
/.pre-commit-config.local.yaml
/.hypothesis/
/eliot.log
/misc/python3/results.xml

View File

@ -1,6 +1,15 @@
repos:
- repo: https://gitlab.com/pycqa/flake8
# TODO: update rev periodically to keep up with tox
rev: 3.8.3
- repo: local
hooks:
- id: flake8
- id: codechecks
name: codechecks
stages: ["commit"]
entry: "tox -e codechecks"
language: system
pass_filenames: false
- id: test
name: test
stages: ["push"]
entry: "make test"
language: system
pass_filenames: false

View File

@ -13,6 +13,8 @@ MAKEFLAGS += --warn-undefined-variables
MAKEFLAGS += --no-builtin-rules
# Local target variables
VCS_HOOK_SAMPLES=$(wildcard .git/hooks/*.sample)
VCS_HOOKS=$(VCS_HOOK_SAMPLES:%.sample=%)
PYTHON=python
export PYTHON
PYFLAKES=flake8
@ -29,10 +31,22 @@ TEST_SUITE=allmydata
default:
@echo "no default target"
.PHONY: install-vcs-hooks
## Install the VCS hooks to run linters on commit and all tests on push
install-vcs-hooks: .git/hooks/pre-commit .git/hooks/pre-push
.PHONY: uninstall-vcs-hooks
## Remove the VCS hooks
uninstall-vcs-hooks: .tox/create-venvs.log
"./$(dir $(<))py36/bin/pre-commit" uninstall || true
"./$(dir $(<))py36/bin/pre-commit" uninstall -t pre-push || true
.PHONY: test
## Run all tests and code reports
test: .tox
tox -p auto
test: .tox/create-venvs.log
# Run codechecks first since it takes the least time to report issues early.
tox --develop -e codechecks
# Run all the test environments in parallel to reduce run-time
tox --develop -p auto -e 'py27,py36,pypy27'
.PHONY: test-venv-coverage
## Run all tests with coverage collection and reporting.
test-venv-coverage:
@ -44,6 +58,10 @@ test-venv-coverage:
$(VIRTUAL_ENV)/bin/coverage xml || true
$(VIRTUAL_ENV)/bin/coverage report
if [ ! -z "$$test_exit" ]; then exit "$$test_exit"; fi
.PHONY: test-py3-all
## Run all tests under Python 3
test-py3-all: .tox/create-venvs.log
tox --develop -e py36 allmydata
# This is necessary only if you want to automatically produce a new
# _version.py file from the current git history (without doing a build).
@ -196,7 +214,7 @@ clean:
rm -f *.pkg
.PHONY: distclean
distclean: clean
distclean: clean uninstall-vcs-hooks
rm -rf src/*.egg-info
rm -f src/allmydata/_version.py
rm -f src/allmydata/_appname.py
@ -240,5 +258,8 @@ upload-tarballs:
src/allmydata/_version.py:
$(MAKE) make-version
.tox: tox.ini setup.py
tox --notest -p all
.tox/create-venvs.log: tox.ini setup.py
tox --notest -p all | tee -a "$(@)"
$(VCS_HOOKS): .tox/create-venvs.log .pre-commit-config.yaml
"./$(dir $(<))py36/bin/pre-commit" install --hook-type $(@:.git/hooks/%=%)

View File

@ -286,7 +286,7 @@ result in a "all tests passed" mesage::
PASSED (skips=7, expectedFailures=3, successes=1176)
__________________________ summary ___________________________________
py27: commands succeeded
congratulations :)
congratulations :)
Common Problems
===============

View File

@ -5,40 +5,85 @@ Developer Guide
Pre-commit Checks
-----------------
This project is configured for use with `pre-commit <https://pre-commit.com>`_ to perform some static code analysis checks. By default, pre-commit behavior is disabled. To enable pre-commit in a local checkout, first install pre-commit (consider using `pipx <https://pipxproject.github.io/pipx/>`_), then install the hooks with ``pre-commit install``.
This project is configured for use with `pre-commit`_ to install `VCS/git hooks`_ which
perform some static code analysis checks and other code checks to catch common errors
before each commit and to run the full self-test suite to find less obvious regressions
before each push to a remote.
For example::
tahoe-lafs $ pre-commit install
pre-commit installed at .git/hooks/pre-commit
tahoe-lafs $ python -c "import pathlib; pathlib.Path('src/allmydata/tabbed.py').write_text('def foo():\\n\\tpass\\n')"
tahoe-lafs $ git add src/allmydata/tabbed.py
tahoe-lafs $ git commit -a -m "Add a file that violates flake8"
flake8...................................................................Failed
- hook id: flake8
- exit code: 1
tahoe-lafs $ make install-vcs-hooks
...
+ ./.tox//py36/bin/pre-commit install --hook-type pre-commit
pre-commit installed at .git/hooks/pre-commit
+ ./.tox//py36/bin/pre-commit install --hook-type pre-push
pre-commit installed at .git/hooks/pre-push
tahoe-lafs $ python -c "import pathlib; pathlib.Path('src/allmydata/tabbed.py').write_text('def foo():\\n\\tpass\\n')"
tahoe-lafs $ git add src/allmydata/tabbed.py
tahoe-lafs $ git commit -a -m "Add a file that violates flake8"
...
codechecks...............................................................Failed
- hook id: codechecks
- exit code: 1
src/allmydata/tabbed.py:2:1: W191 indentation contains tabs
GLOB sdist-make: ./tahoe-lafs/setup.py
codechecks inst-nodeps: ...
codechecks installed: ...
codechecks run-test-pre: PYTHONHASHSEED='...'
codechecks run-test: commands[0] | flake8 src static misc setup.py
src/allmydata/tabbed.py:2:1: W191 indentation contains tabs
ERROR: InvocationError for command ./tahoe-lafs/.tox/codechecks/bin/flake8 src static misc setup.py (exited with code 1)
___________________________________ summary ____________________________________
ERROR: codechecks: commands failed
...
To uninstall::
tahoe-lafs $ pre-commit uninstall
pre-commit uninstalled
tahoe-lafs $ make uninstall-vcs-hooks
...
+ ./.tox/py36/bin/pre-commit uninstall
pre-commit uninstalled
+ ./.tox/py36/bin/pre-commit uninstall -t pre-push
pre-push uninstalled
Note that running the full self-test suite takes several minutes so expect pushing to
take some time. If you can't or don't want to wait for the hooks in some cases, use the
``--no-verify`` option to ``$ git commit ...`` or ``$ git push ...``. Alternatively,
see the `pre-commit`_ documentation and CLI help output and use the committed
`pre-commit configuration`_ as a starting point to write a local, uncommitted
``../.pre-commit-config.local.yaml`` configuration to use instead. For example::
tahoe-lafs $ ./.tox/py36/bin/pre-commit --help
tahoe-lafs $ ./.tox/py36/bin/pre-commit instll --help
tahoe-lafs $ cp "./.pre-commit-config.yaml" "./.pre-commit-config.local.yaml"
tahoe-lafs $ editor "./.pre-commit-config.local.yaml"
...
tahoe-lafs $ ./.tox/py36/bin/pre-commit install -c "./.pre-commit-config.local.yaml" -t pre-push
pre-commit installed at .git/hooks/pre-push
tahoe-lafs $ git commit -a -m "Add a file that violates flake8"
[3398.pre-commit 29f8f43d2] Add a file that violates flake8
1 file changed, 2 insertions(+)
create mode 100644 src/allmydata/tabbed.py
tahoe-lafs $ git push
...
codechecks...............................................................Failed
- hook id: codechecks
- exit code: 1
GLOB sdist-make: ./tahoe-lafs/setup.py
codechecks inst-nodeps: ...
codechecks installed: ...
codechecks run-test-pre: PYTHONHASHSEED='...'
codechecks run-test: commands[0] | flake8 src static misc setup.py
src/allmydata/tabbed.py:2:1: W191 indentation contains tabs
ERROR: InvocationError for command ./tahoe-lafs/.tox/codechecks/bin/flake8 src static misc setup.py (exited with code 1)
___________________________________ summary ____________________________________
ERROR: codechecks: commands failed
...
error: failed to push some refs to 'github.com:jaraco/tahoe-lafs.git'
Some find running linters on every commit to be a nuisance. To avoid the checks triggering during commits, but to check before pushing to the CI, install the hook for pre-push instead::
tahoe-lafs $ pre-commit install -t pre-push
pre-commit installed at .git/hooks/pre-push
tahoe-lafs $ git commit -a -m "Add a file that violates flake8"
[3398.pre-commit 29f8f43d2] Add a file that violates flake8
1 file changed, 2 insertions(+)
create mode 100644 src/allmydata/tabbed.py
tahoe-lafs $ git push
flake8...................................................................Failed
- hook id: flake8
- exit code: 1
src/allmydata/tabbed.py:2:1: W191 indentation contains tabs
error: failed to push some refs to 'github.com:jaraco/tahoe-lafs.git'
.. _`pre-commit`: https://pre-commit.com
.. _`VCS/git hooks`: `pre-commit`_
.. _`pre-commit configuration`: ../.pre-commit-config.yaml

1
newsfragments/3442.minor Normal file
View File

@ -0,0 +1 @@
Minor test runner improvements and docs.

0
newsfragments/3451.minor Normal file
View File

0
newsfragments/3452.minor Normal file
View File

View File

@ -0,0 +1,13 @@
"""
Ported to Python 3.
"""
from __future__ import unicode_literals
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from future.utils import PY2
if PY2:
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401

View File

@ -1,3 +1,15 @@
"""
Ported to Python 3.
"""
from __future__ import unicode_literals
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from future.utils import PY2
if PY2:
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
(AVAILABLE, PENDING, OVERDUE, COMPLETE, CORRUPT, DEAD, BADSEGNUM) = \
("AVAILABLE", "PENDING", "OVERDUE", "COMPLETE", "CORRUPT", "DEAD", "BADSEGNUM")

View File

@ -1,4 +1,15 @@
"""
Ported to Python 3.
"""
from __future__ import print_function
from __future__ import absolute_import
from __future__ import division
from __future__ import unicode_literals
from future.utils import PY2
if PY2:
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
from twisted.python.failure import Failure
from foolscap.api import eventually
@ -222,7 +233,7 @@ class SegmentFetcher(object):
# add_shares() or no_more_shares() later.
def _cancel_all_requests(self):
for o in self._share_observers.values():
for o in list(self._share_observers.values()):
o.cancel()
self._share_observers = {}

View File

@ -1,3 +1,14 @@
"""
Ported to Python 3.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
from future.utils import PY2
if PY2:
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
import time
now = time.time

View File

@ -1,3 +1,14 @@
"""
Ported to Python 3.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
from future.utils import PY2
if PY2:
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
import time
now = time.time

View File

@ -1,3 +1,14 @@
"""
Ported to Python 3.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
from future.utils import PY2
if PY2:
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
import time
now = time.time

View File

@ -1,3 +1,14 @@
"""
Ported to Python 3.
"""
from __future__ import division
from __future__ import absolute_import
from __future__ import print_function
from __future__ import unicode_literals
from future.utils import PY2
if PY2:
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
import itertools
from zope.interface import implementer
@ -9,8 +20,8 @@ class ReadEvent(object):
self._ev = ev
self._ds = ds
def update(self, bytes, decrypttime, pausetime):
self._ev["bytes_returned"] += bytes
def update(self, bytes_returned, decrypttime, pausetime):
self._ev["bytes_returned"] += bytes_returned
self._ev["decrypt_time"] += decrypttime
self._ev["paused_time"] += pausetime
@ -257,7 +268,7 @@ class DownloadStatus(object):
# else ignore completed requests
if not total_outstanding:
return 1.0
return 1.0 * total_received / total_outstanding
return total_received / total_outstanding
def using_helper(self):
return False

View File

@ -1334,6 +1334,21 @@ class FakeNode(object):
class Selection(unittest.TestCase):
def test_failure(self):
"""If the fetch loop fails, it tell the Node the fetch failed."""
node = FakeNode()
# Simulate a failure:
node.get_num_segments = lambda: 1/0
sf = SegmentFetcher(node, 0, 3, None)
sf.add_shares([])
d = flushEventualQueue()
def _check1(ign):
[_] = self.flushLoggedErrors(ZeroDivisionError)
self.failUnless(node.failed)
self.failUnless(node.failed.check(ZeroDivisionError))
d.addCallback(_check1)
return d
def test_no_shares(self):
node = FakeNode()
sf = SegmentFetcher(node, 0, 3, None)

View File

@ -32,6 +32,13 @@ PORTED_MODULES = [
"allmydata.crypto.rsa",
"allmydata.crypto.util",
"allmydata.hashtree",
"allmydata.immutable.downloader",
"allmydata.immutable.downloader.common",
"allmydata.immutable.downloader.fetcher",
"allmydata.immutable.downloader.finder",
"allmydata.immutable.downloader.node",
"allmydata.immutable.downloader.segmentation",
"allmydata.immutable.downloader.status",
"allmydata.immutable.happiness_upload",
"allmydata.immutable.literal",
"allmydata.interfaces",

12
tox.ini
View File

@ -7,7 +7,7 @@
twisted = 1
[tox]
envlist = {py27,pypy27,py36}{-coverage,}
envlist = codechecks,py27,py36,pypy27
minversion = 2.4
[testenv]
@ -37,6 +37,8 @@ deps =
# regressions in new releases of this package that cause us the kind of
# suffering we're trying to avoid with the above pins.
certifi
# VCS hooks support
py36,!coverage: pre-commit
# We add usedevelop=False because testing against a true installation gives
# more useful results.
@ -67,6 +69,7 @@ commands =
coverage: coverage run -m twisted.trial {env:TAHOE_LAFS_TRIAL_ARGS:--rterrors --reporter=timing} {posargs:{env:TEST_SUITE}}
coverage: coverage combine
coverage: coverage xml
coverage: coverage report
[testenv:integration]
@ -85,6 +88,13 @@ basepython = python2.7
passenv = HOME
whitelist_externals =
/bin/mv
setenv =
# Workaround an error when towncrier is run under the VCS hook,
# https://stackoverflow.com/a/4027726/624787:
# File "/home/rpatterson/src/work/sfu/tahoe-lafs/.tox/codechecks/lib/python2.7/site-packages/towncrier/check.py", line 44, in __main
# .decode(getattr(sys.stdout, "encoding", "utf8"))
# `TypeError: decode() argument 1 must be string, not None`
PYTHONIOENCODING=utf_8
commands =
flake8 src static misc setup.py
python misc/coding_tools/check-umids.py src