From bd2f43884358b37259861870098538eea35b4eff Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 17 Jul 2020 13:48:08 -0400 Subject: [PATCH 01/79] Replace the TravisCI Python 3.6 config with CircleCI Python 3.6 config --- .circleci/config.yml | 12 +++++++++++- .travis.yml | 40 ---------------------------------------- 2 files changed, 11 insertions(+), 41 deletions(-) delete mode 100644 .travis.yml diff --git a/.circleci/config.yml b/.circleci/config.yml index d0f596fc7..9654905d7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -30,6 +30,9 @@ workflows: # Test against PyPy 2.7 - "pypy2.7-buster" + # Just one Python 3.6 configuration while the port is in-progress. + - "python3.6" + # Other assorted tasks and configurations - "lint" - "pyinstaller" @@ -261,13 +264,20 @@ jobs: user: "nobody" - ubuntu-18.04: + ubuntu-18.04: &UBUNTU_18_04 <<: *DEBIAN docker: - image: "tahoelafsci/ubuntu:18.04" user: "nobody" + python3.6: + <<: *UBUNTU_18_04 + environment: + <<: *UTF_8_ENVIRONMENT + TAHOE_LAFS_TOX_ENVIRONMENT: "py36" + + centos-8: &RHEL_DERIV docker: - image: "tahoelafsci/centos:8" diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 40f351ae6..000000000 --- a/.travis.yml +++ /dev/null @@ -1,40 +0,0 @@ -sudo: false -language: python -cache: pip -dist: xenial -before_cache: - - rm -f $HOME/.cache/pip/log/debug.log -git: - depth: 1000 - -env: - global: - - TAHOE_LAFS_HYPOTHESIS_PROFILE=ci - -install: - - pip install --upgrade tox setuptools virtualenv - - echo $PATH; which python; which pip; which tox - - python misc/build_helpers/show-tool-versions.py - -script: - - | - set -eo pipefail - tox -e ${T} - -notifications: - email: false - irc: - channels: "chat.freenode.net#tahoe-lafs" - on_success: always # for testing - on_failure: always - template: - - "%{repository}#%{build_number} [%{branch}: %{commit} by %{author}] %{message}" - - "Changes: %{compare_url} | Details: %{build_url}" - -matrix: - include: - - os: linux - python: '3.6' - env: T=py36 - - fast_finish: true From b0a5a65f4f8494c8d8d19014a31ae67fb4d9fcc9 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 17 Jul 2020 16:23:42 -0400 Subject: [PATCH 02/79] Attempt to enable a Python 3 CircleCI job --- .circleci/Dockerfile.centos | 7 ++++--- .circleci/Dockerfile.debian | 7 ++++--- .circleci/Dockerfile.fedora | 7 ++++--- .circleci/Dockerfile.slackware | 5 +++-- .circleci/Dockerfile.ubuntu | 7 ++++--- .circleci/config.yml | 22 ++++++++++++++++++++++ 6 files changed, 41 insertions(+), 14 deletions(-) diff --git a/.circleci/Dockerfile.centos b/.circleci/Dockerfile.centos index febb61545..632e13137 100644 --- a/.circleci/Dockerfile.centos +++ b/.circleci/Dockerfile.centos @@ -1,4 +1,5 @@ ARG TAG +ARG PYTHON_VERSION FROM centos:${TAG} ENV WHEELHOUSE_PATH /tmp/wheelhouse @@ -11,8 +12,8 @@ RUN yum install --assumeyes \ git \ sudo \ make automake gcc gcc-c++ \ - python2 \ - python2-devel \ + python${PYTHON_VERSION} \ + python${PYTHON_VERSION}-devel \ libffi-devel \ openssl-devel \ libyaml \ @@ -23,4 +24,4 @@ RUN yum install --assumeyes \ # *update* this checkout on each job run, saving us more time per-job. COPY . ${BUILD_SRC_ROOT} -RUN "${BUILD_SRC_ROOT}"/.circleci/prepare-image.sh "${WHEELHOUSE_PATH}" "${VIRTUALENV_PATH}" "${BUILD_SRC_ROOT}" "python2.7" +RUN "${BUILD_SRC_ROOT}"/.circleci/prepare-image.sh "${WHEELHOUSE_PATH}" "${VIRTUALENV_PATH}" "${BUILD_SRC_ROOT}" "python${PYTHON_VERSION}" diff --git a/.circleci/Dockerfile.debian b/.circleci/Dockerfile.debian index de16148e0..394495e17 100644 --- a/.circleci/Dockerfile.debian +++ b/.circleci/Dockerfile.debian @@ -1,4 +1,5 @@ ARG TAG +ARG PYTHON_VERSION FROM debian:${TAG} ENV WHEELHOUSE_PATH /tmp/wheelhouse @@ -12,8 +13,8 @@ RUN apt-get --quiet update && \ lsb-release \ sudo \ build-essential \ - python2.7 \ - python2.7-dev \ + python${PYTHON_VERSION} \ + python${PYTHON_VERSION}-dev \ libffi-dev \ libssl-dev \ libyaml-dev \ @@ -23,7 +24,7 @@ RUN apt-get --quiet update && \ # *update* this checkout on each job run, saving us more time per-job. COPY . ${BUILD_SRC_ROOT} -RUN "${BUILD_SRC_ROOT}"/.circleci/prepare-image.sh "${WHEELHOUSE_PATH}" "${VIRTUALENV_PATH}" "${BUILD_SRC_ROOT}" "python2.7" +RUN "${BUILD_SRC_ROOT}"/.circleci/prepare-image.sh "${WHEELHOUSE_PATH}" "${VIRTUALENV_PATH}" "${BUILD_SRC_ROOT}" "python${PYTHON_VERSION}" # Only the integration tests currently need this but it doesn't hurt to always # have it present and it's simpler than building a whole extra image just for diff --git a/.circleci/Dockerfile.fedora b/.circleci/Dockerfile.fedora index 6ad22d676..f17140f3e 100644 --- a/.circleci/Dockerfile.fedora +++ b/.circleci/Dockerfile.fedora @@ -1,4 +1,5 @@ ARG TAG +ARG PYTHON_VERSION FROM fedora:${TAG} ENV WHEELHOUSE_PATH /tmp/wheelhouse @@ -11,8 +12,8 @@ RUN yum install --assumeyes \ git \ sudo \ make automake gcc gcc-c++ \ - python \ - python-devel \ + python${PYTHON_VERSION} \ + python${PYTHON_VERSION}-devel \ libffi-devel \ openssl-devel \ libyaml-devel \ @@ -23,4 +24,4 @@ RUN yum install --assumeyes \ # *update* this checkout on each job run, saving us more time per-job. COPY . ${BUILD_SRC_ROOT} -RUN "${BUILD_SRC_ROOT}"/.circleci/prepare-image.sh "${WHEELHOUSE_PATH}" "${VIRTUALENV_PATH}" "${BUILD_SRC_ROOT}" "python2.7" +RUN "${BUILD_SRC_ROOT}"/.circleci/prepare-image.sh "${WHEELHOUSE_PATH}" "${VIRTUALENV_PATH}" "${BUILD_SRC_ROOT}" "python${PYTHON_VERSION}" diff --git a/.circleci/Dockerfile.slackware b/.circleci/Dockerfile.slackware index 73ba6b32d..97ee07609 100644 --- a/.circleci/Dockerfile.slackware +++ b/.circleci/Dockerfile.slackware @@ -1,4 +1,5 @@ ARG TAG +ARG PYTHON_VERSION FROM vbatts/slackware:${TAG} ENV WHEELHOUSE_PATH /tmp/wheelhouse @@ -23,7 +24,7 @@ RUN slackpkg update && \ binutils-2 \ gcc-5 \ gcc-g++-5 \ - python-2 \ + python-${PYTHON_VERSION} \ libffi-3 \ libyaml-0 \ sqlite-3 \ @@ -46,4 +47,4 @@ RUN slackpkg install \ # *update* this checkout on each job run, saving us more time per-job. COPY . ${BUILD_SRC_ROOT} -RUN "${BUILD_SRC_ROOT}"/.circleci/prepare-image.sh "${WHEELHOUSE_PATH}" "${VIRTUALENV_PATH}" "${BUILD_SRC_ROOT}" "python2.7" +RUN "${BUILD_SRC_ROOT}"/.circleci/prepare-image.sh "${WHEELHOUSE_PATH}" "${VIRTUALENV_PATH}" "${BUILD_SRC_ROOT}" "python${PYTHON_VERSION}" diff --git a/.circleci/Dockerfile.ubuntu b/.circleci/Dockerfile.ubuntu index 882dfe308..4ca2319a9 100644 --- a/.circleci/Dockerfile.ubuntu +++ b/.circleci/Dockerfile.ubuntu @@ -1,4 +1,5 @@ ARG TAG +ARG PYTHON_VERSION FROM ubuntu:${TAG} ENV WHEELHOUSE_PATH /tmp/wheelhouse @@ -13,8 +14,8 @@ RUN apt-get --quiet update && \ apt-get --quiet --yes install \ sudo \ build-essential \ - python2.7 \ - python2.7-dev \ + python${PYTHON_VERSION} \ + python${PYTHON_VERSION}-dev \ libffi-dev \ libssl-dev \ libyaml-dev \ @@ -26,4 +27,4 @@ RUN apt-get --quiet update && \ # *update* this checkout on each job run, saving us more time per-job. COPY . ${BUILD_SRC_ROOT} -RUN "${BUILD_SRC_ROOT}"/.circleci/prepare-image.sh "${WHEELHOUSE_PATH}" "${VIRTUALENV_PATH}" "${BUILD_SRC_ROOT}" "python2.7" +RUN "${BUILD_SRC_ROOT}"/.circleci/prepare-image.sh "${WHEELHOUSE_PATH}" "${VIRTUALENV_PATH}" "${BUILD_SRC_ROOT}" "python${PYTHON_VERSION}" diff --git a/.circleci/config.yml b/.circleci/config.yml index 9654905d7..dd423e2af 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -73,6 +73,7 @@ workflows: - "build-image-centos-8" - "build-image-slackware-14.2" - "build-image-pypy-2.7-buster" + - "build-image-python36-ubuntu" jobs: @@ -449,6 +450,7 @@ jobs: docker \ build \ --build-arg TAG=${TAG} \ + --build-arg PYTHON_VERSION=${PYTHON_VERSION:-python2} -t tahoelafsci/${DISTRO}:${TAG} \ -f ~/project/.circleci/Dockerfile.${DISTRO} \ ~/project/ @@ -464,6 +466,7 @@ jobs: environment: DISTRO: "debian" TAG: "8" + PYTHON_VERSION: "2.7" build-image-debian-9: @@ -472,6 +475,7 @@ jobs: environment: DISTRO: "debian" TAG: "9" + PYTHON_VERSION: "2.7" build-image-ubuntu-16.04: @@ -480,6 +484,7 @@ jobs: environment: DISTRO: "ubuntu" TAG: "16.04" + PYTHON_VERSION: "2.7" build-image-ubuntu-18.04: @@ -488,6 +493,16 @@ jobs: environment: DISTRO: "ubuntu" TAG: "18.04" + PYTHON_VERSION: "2.7" + + + build-image-python36-ubuntu: + <<: *BUILD_IMAGE + + environment: + DISTRO: "ubuntu" + TAG: "18.04" + PYTHON_VERSION: "3" build-image-centos-8: @@ -496,6 +511,7 @@ jobs: environment: DISTRO: "centos" TAG: "8" + PYTHON_VERSION: "2" build-image-fedora-28: @@ -504,6 +520,8 @@ jobs: environment: DISTRO: "fedora" TAG: "28" + # The default on Fedora (this version anyway) is still Python 2. + PYTHON_VERSION: "" build-image-fedora-29: @@ -520,6 +538,7 @@ jobs: environment: DISTRO: "slackware" TAG: "14.2" + PYTHON_VERSION: "2" build-image-pypy-2.7-buster: @@ -528,3 +547,6 @@ jobs: environment: DISTRO: "pypy" TAG: "2.7-buster" + # We only have Python 2 for PyPy right now so there's no support for + # setting up PyPy 3 in the image building toolchain.. + PYTHON_VERSION: "" From 9c16a0554e7ee0f67f3ffcd757d3c475da84ef26 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 29 Jul 2020 14:38:58 -0400 Subject: [PATCH 03/79] Get rid of unused code. --- src/allmydata/test/test_util.py | 77 +-------------------------------- src/allmydata/util/limiter.py | 40 ----------------- 2 files changed, 1 insertion(+), 116 deletions(-) delete mode 100644 src/allmydata/util/limiter.py diff --git a/src/allmydata/test/test_util.py b/src/allmydata/test/test_util.py index 1e9dd0f49..330abebe5 100644 --- a/src/allmydata/test/test_util.py +++ b/src/allmydata/test/test_util.py @@ -10,7 +10,7 @@ from twisted.python.failure import Failure from allmydata.util import idlib, mathutil from allmydata.util import fileutil -from allmydata.util import limiter, pollmixin +from allmydata.util import pollmixin from allmydata.util import yamlutil from allmydata.util import log as tahoe_log from allmydata.util.fileutil import EncryptedTemporaryFile @@ -429,81 +429,6 @@ class PollMixinTests(unittest.TestCase): return d -class Limiter(unittest.TestCase): - - def job(self, i, foo): - self.calls.append( (i, foo) ) - self.simultaneous += 1 - self.peak_simultaneous = max(self.simultaneous, self.peak_simultaneous) - d = defer.Deferred() - def _done(): - self.simultaneous -= 1 - d.callback("done %d" % i) - reactor.callLater(1.0, _done) - return d - - def bad_job(self, i, foo): - raise ValueError("bad_job %d" % i) - - def test_limiter(self): - self.calls = [] - self.simultaneous = 0 - self.peak_simultaneous = 0 - l = limiter.ConcurrencyLimiter() - dl = [] - for i in range(20): - dl.append(l.add(self.job, i, foo=str(i))) - d = defer.DeferredList(dl, fireOnOneErrback=True) - def _done(res): - self.failUnlessEqual(self.simultaneous, 0) - self.failUnless(self.peak_simultaneous <= 10) - self.failUnlessEqual(len(self.calls), 20) - for i in range(20): - self.failUnless( (i, str(i)) in self.calls) - d.addCallback(_done) - return d - - def test_errors(self): - self.calls = [] - self.simultaneous = 0 - self.peak_simultaneous = 0 - l = limiter.ConcurrencyLimiter() - dl = [] - for i in range(20): - dl.append(l.add(self.job, i, foo=str(i))) - d2 = l.add(self.bad_job, 21, "21") - d = defer.DeferredList(dl, fireOnOneErrback=True) - def _most_done(res): - results = [] - for (success, result) in res: - self.failUnlessEqual(success, True) - results.append(result) - results.sort() - expected_results = ["done %d" % i for i in range(20)] - expected_results.sort() - self.failUnlessEqual(results, expected_results) - self.failUnless(self.peak_simultaneous <= 10) - self.failUnlessEqual(len(self.calls), 20) - for i in range(20): - self.failUnless( (i, str(i)) in self.calls) - def _good(res): - self.fail("should have failed, not got %s" % (res,)) - def _err(f): - f.trap(ValueError) - self.failUnless("bad_job 21" in str(f)) - d2.addCallbacks(_good, _err) - return d2 - d.addCallback(_most_done) - def _all_done(res): - self.failUnlessEqual(self.simultaneous, 0) - self.failUnless(self.peak_simultaneous <= 10) - self.failUnlessEqual(len(self.calls), 20) - for i in range(20): - self.failUnless( (i, str(i)) in self.calls) - d.addCallback(_all_done) - return d - - ctr = [0] class EqButNotIs(object): def __init__(self, x): diff --git a/src/allmydata/util/limiter.py b/src/allmydata/util/limiter.py deleted file mode 100644 index 0391ede11..000000000 --- a/src/allmydata/util/limiter.py +++ /dev/null @@ -1,40 +0,0 @@ - -from twisted.internet import defer -from foolscap.api import eventually - -class ConcurrencyLimiter(object): - """I implement a basic concurrency limiter. Add work to it in the form of - (callable, args, kwargs) tuples. No more than LIMIT callables will be - outstanding at any one time. - """ - - def __init__(self, limit=10): - self.limit = limit - self.pending = [] - self.active = 0 - - def __repr__(self): - return "" % (self.active, len(self.pending), - self.limit) - - def add(self, cb, *args, **kwargs): - d = defer.Deferred() - task = (cb, args, kwargs, d) - self.pending.append(task) - self.maybe_start_task() - return d - - def maybe_start_task(self): - if self.active >= self.limit: - return - if not self.pending: - return - (cb, args, kwargs, done_d) = self.pending.pop(0) - self.active += 1 - d = defer.maybeDeferred(cb, *args, **kwargs) - d.addBoth(self._done, done_d) - - def _done(self, res, done_d): - self.active -= 1 - eventually(done_d.callback, res) - eventually(self.maybe_start_task) From 0a1e365ed1ea1832dcf58e0b3fc934ad2d4c757a Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 29 Jul 2020 14:46:49 -0400 Subject: [PATCH 04/79] Tests for gcutil. --- src/allmydata/test/test_iputil.py | 33 ++++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/src/allmydata/test/test_iputil.py b/src/allmydata/test/test_iputil.py index c6caed7f9..06ab13f39 100644 --- a/src/allmydata/test/test_iputil.py +++ b/src/allmydata/test/test_iputil.py @@ -14,6 +14,7 @@ if PY2: from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 import re, errno, subprocess, os, socket +import gc from twisted.trial import unittest @@ -21,7 +22,7 @@ from tenacity import retry, stop_after_attempt from foolscap.api import Tub -from allmydata.util import iputil +from allmydata.util import iputil, gcutil import allmydata.test.common_py3 as testutil from allmydata.util.namespace import Namespace @@ -228,3 +229,33 @@ class ListenOnUsed(unittest.TestCase): s.close() port2 = iputil.listenOnUnused(tub, port) self.assertEqual(port, port2) + + +class GcUtil(unittest.TestCase): + """Tests for allmydata.util.gcutil, which is used only by listenOnUnused.""" + + def test_gc_after_allocations(self): + """The resource tracker triggers allocations every 26 allocations.""" + collections = [] + self.patch(gc, "collect", lambda: collections.append(1)) + for _ in range(2): + for _ in range(25): + gcutil.fileDescriptorResource.allocate() + self.assertEqual(len(collections), 0) + gcutil.fileDescriptorResource.allocate() + self.assertEqual(len(collections), 1) + del collections[:] + + def test_release_delays_gc(self): + """Releasing a file descriptor resource delays GC collection.""" + collections = [] + self.patch(gc, "collect", lambda: collections.append(1)) + for _ in range(2): + gcutil.fileDescriptorResource.allocate() + for _ in range(3): + gcutil.fileDescriptorResource.release() + for _ in range(25): + gcutil.fileDescriptorResource.allocate() + self.assertEqual(len(collections), 0) + gcutil.fileDescriptorResource.allocate() + self.assertEqual(len(collections), 1) From 5e308260b105865db1f472ed12524ccf7b18ae14 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 29 Jul 2020 14:49:48 -0400 Subject: [PATCH 05/79] Port to Python 3. --- newsfragments/3361.minor | 0 src/allmydata/util/_python3.py | 1 + src/allmydata/util/gcutil.py | 10 ++++++++++ 3 files changed, 11 insertions(+) create mode 100644 newsfragments/3361.minor diff --git a/newsfragments/3361.minor b/newsfragments/3361.minor new file mode 100644 index 000000000..e69de29bb diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 8a75a8fc5..b91e079f8 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -22,6 +22,7 @@ PORTED_MODULES = [ "allmydata.util.base62", "allmydata.util.deferredutil", "allmydata.util.dictutil", + "allmydata.util.gcutil", "allmydata.util.hashutil", "allmydata.util.humanreadable", "allmydata.util.iputil", diff --git a/src/allmydata/util/gcutil.py b/src/allmydata/util/gcutil.py index 8fb3a64c9..ade95341f 100644 --- a/src/allmydata/util/gcutil.py +++ b/src/allmydata/util/gcutil.py @@ -7,7 +7,17 @@ Helpers for managing garbage collection. a result. Register allocation and release of *bare* file descriptors with this object (file objects, socket objects, etc, have their own integration with the garbage collector and don't need to bother with this). + +Ported to Python 3. """ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 __all__ = [ "fileDescriptorResource", From 3a6587971aed1656671939ae745fa4a20897807f Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 29 Jul 2020 14:51:11 -0400 Subject: [PATCH 06/79] Ratchet. --- misc/python3/ratchet-passing | 2 ++ 1 file changed, 2 insertions(+) diff --git a/misc/python3/ratchet-passing b/misc/python3/ratchet-passing index b9beb360e..d1ea2dfa2 100644 --- a/misc/python3/ratchet-passing +++ b/misc/python3/ratchet-passing @@ -46,6 +46,8 @@ allmydata.test.test_hashutil.HashUtilTests.test_sha256d allmydata.test.test_hashutil.HashUtilTests.test_sha256d_truncated allmydata.test.test_hashutil.HashUtilTests.test_timing_safe_compare allmydata.test.test_humanreadable.HumanReadable.test_repr +allmydata.test.test_iputil.GcUtil.test_gc_after_allocations +allmydata.test.test_iputil.GcUtil.test_release_delays_gc allmydata.test.test_iputil.ListAddresses.test_get_local_ip_for allmydata.test.test_iputil.ListAddresses.test_list_async allmydata.test.test_iputil.ListAddresses.test_list_async_mock_cygwin From 34e3052814ca409ffef339dd801ea9816bacdab8 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 3 Aug 2020 10:43:21 -0400 Subject: [PATCH 07/79] Don't use global state for testing. --- src/allmydata/test/test_iputil.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/allmydata/test/test_iputil.py b/src/allmydata/test/test_iputil.py index 06ab13f39..d3262a9ea 100644 --- a/src/allmydata/test/test_iputil.py +++ b/src/allmydata/test/test_iputil.py @@ -236,26 +236,28 @@ class GcUtil(unittest.TestCase): def test_gc_after_allocations(self): """The resource tracker triggers allocations every 26 allocations.""" + tracker = gcutil._ResourceTracker() collections = [] self.patch(gc, "collect", lambda: collections.append(1)) for _ in range(2): for _ in range(25): - gcutil.fileDescriptorResource.allocate() + tracker.allocate() self.assertEqual(len(collections), 0) - gcutil.fileDescriptorResource.allocate() + tracker.allocate() self.assertEqual(len(collections), 1) del collections[:] def test_release_delays_gc(self): """Releasing a file descriptor resource delays GC collection.""" + tracker = gcutil._ResourceTracker() collections = [] self.patch(gc, "collect", lambda: collections.append(1)) for _ in range(2): - gcutil.fileDescriptorResource.allocate() + tracker.allocate() for _ in range(3): - gcutil.fileDescriptorResource.release() + tracker.release() for _ in range(25): - gcutil.fileDescriptorResource.allocate() + tracker.allocate() self.assertEqual(len(collections), 0) - gcutil.fileDescriptorResource.allocate() + tracker.allocate() self.assertEqual(len(collections), 1) From 3330737656ce6cbe79e4210774429d125aa15fef Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 3 Aug 2020 10:56:47 -0400 Subject: [PATCH 08/79] Be a bit stricter about expected outputs. --- src/allmydata/test/test_crypto.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/allmydata/test/test_crypto.py b/src/allmydata/test/test_crypto.py index 53ba344a4..1cf55ca0f 100644 --- a/src/allmydata/test/test_crypto.py +++ b/src/allmydata/test/test_crypto.py @@ -1,3 +1,5 @@ +from future.utils import native_bytes + import six import unittest @@ -283,7 +285,7 @@ class TestEd25519(unittest.TestCase): private_key, public_key = ed25519.create_signing_keypair() private_key_str = ed25519.string_from_signing_key(private_key) - self.assertIsInstance(private_key_str, six.string_types) + self.assertIsInstance(private_key_str, native_bytes) private_key2, public_key2 = ed25519.signing_keypair_from_string(private_key_str) @@ -299,7 +301,7 @@ class TestEd25519(unittest.TestCase): # ditto, but for the verifying keys public_key_str = ed25519.string_from_verifying_key(public_key) - self.assertIsInstance(public_key_str, six.string_types) + self.assertIsInstance(public_key_str, native_bytes) public_key2 = ed25519.verifying_key_from_string(public_key_str) self.assertEqual( @@ -403,7 +405,7 @@ class TestRsa(unittest.TestCase): priv_key, pub_key = rsa.create_signing_keypair(2048) priv_key_str = rsa.der_string_from_signing_key(priv_key) - self.assertIsInstance(priv_key_str, six.string_types) + self.assertIsInstance(priv_key_str, native_bytes) priv_key2, pub_key2 = rsa.create_signing_keypair_from_string(priv_key_str) From 5815547661247ec021103a9a559c3de1fbc78728 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 3 Aug 2020 10:58:37 -0400 Subject: [PATCH 09/79] Remove an unnecessary conversion that breaks Python 3. --- src/allmydata/test/test_crypto.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/allmydata/test/test_crypto.py b/src/allmydata/test/test_crypto.py index 1cf55ca0f..1e9402370 100644 --- a/src/allmydata/test/test_crypto.py +++ b/src/allmydata/test/test_crypto.py @@ -39,17 +39,18 @@ class TestRegression(unittest.TestCase): # priv = rsa.generate(2048) # priv_str = b64encode(priv.serialize()) # pub_str = b64encode(priv.get_verifying_key().serialize()) - RSA_2048_PRIV_KEY = six.b(b64decode(f.read().strip())) + RSA_2048_PRIV_KEY = b64decode(f.read().strip()) + assert isinstance(RSA_2048_PRIV_KEY, native_bytes) with RESOURCE_DIR.child('pycryptopp-rsa-2048-sig.txt').open('r') as f: # Signature created using `RSA_2048_PRIV_KEY` via: # # sig = priv.sign(b'test') - RSA_2048_SIG = six.b(b64decode(f.read().strip())) + RSA_2048_SIG = b64decode(f.read().strip()) with RESOURCE_DIR.child('pycryptopp-rsa-2048-pub.txt').open('r') as f: # The public key corresponding to `RSA_2048_PRIV_KEY`. - RSA_2048_PUB_KEY = six.b(b64decode(f.read().strip())) + RSA_2048_PUB_KEY = b64decode(f.read().strip()) def test_old_start_up_test(self): """ From cd50ac434a556c3784d726051bb460b400f58fde Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 3 Aug 2020 10:59:51 -0400 Subject: [PATCH 10/79] Port to Python 3. --- src/allmydata/test/test_crypto.py | 9 +++++++++ src/allmydata/util/_python3.py | 1 + 2 files changed, 10 insertions(+) diff --git a/src/allmydata/test/test_crypto.py b/src/allmydata/test/test_crypto.py index 1e9402370..7f6271ef3 100644 --- a/src/allmydata/test/test_crypto.py +++ b/src/allmydata/test/test_crypto.py @@ -1,3 +1,12 @@ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from future.utils import native_bytes import six diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 8a75a8fc5..f96a6f1e4 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -42,6 +42,7 @@ PORTED_TEST_MODULES = [ "allmydata.test.test_abbreviate", "allmydata.test.test_base32", "allmydata.test.test_base62", + "allmydata.test.test_crypto", "allmydata.test.test_deferredutil", "allmydata.test.test_dictutil", "allmydata.test.test_hashtree", From d71951ecf1313926efc291664c182af5d421343b Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 3 Aug 2020 11:00:09 -0400 Subject: [PATCH 11/79] News fragment. --- newsfragments/3364.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3364.minor diff --git a/newsfragments/3364.minor b/newsfragments/3364.minor new file mode 100644 index 000000000..e69de29bb From 24729dbd610d4a7a15855c4934c627e4ead97b59 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 3 Aug 2020 11:06:46 -0400 Subject: [PATCH 12/79] Port to Python 3. --- src/allmydata/crypto/__init__.py | 11 +++++++++++ src/allmydata/crypto/aes.py | 10 ++++++++++ src/allmydata/crypto/ed25519.py | 11 +++++++++++ src/allmydata/crypto/error.py | 10 ++++++++++ src/allmydata/crypto/rsa.py | 11 ++++++++++- src/allmydata/crypto/util.py | 10 ++++++++++ src/allmydata/util/_python3.py | 6 ++++++ 7 files changed, 68 insertions(+), 1 deletion(-) diff --git a/src/allmydata/crypto/__init__.py b/src/allmydata/crypto/__init__.py index ee92f223a..4eb753b57 100644 --- a/src/allmydata/crypto/__init__.py +++ b/src/allmydata/crypto/__init__.py @@ -5,4 +5,15 @@ For the most part, these functions use and return objects that are documented in the `cryptography` library -- however, code inside Tahoe should only use these functions and not rely on features of any objects that `cryptography` documents. + +Ported to Python 3. """ + +from __future__ import unicode_literals +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from future.utils import PY2 +if PY2: + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 diff --git a/src/allmydata/crypto/aes.py b/src/allmydata/crypto/aes.py index 4194c63df..196d751bd 100644 --- a/src/allmydata/crypto/aes.py +++ b/src/allmydata/crypto/aes.py @@ -6,7 +6,17 @@ These functions use and return objects that are documented in the `cryptography` library -- however, code inside Tahoe should only use functions from allmydata.crypto.aes and not rely on features of any objects that `cryptography` documents. + +Ported to Python 3. """ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 import six diff --git a/src/allmydata/crypto/ed25519.py b/src/allmydata/crypto/ed25519.py index 37e305d19..424dc23fa 100644 --- a/src/allmydata/crypto/ed25519.py +++ b/src/allmydata/crypto/ed25519.py @@ -13,7 +13,18 @@ cut-and-pasteability. The base62 encoding is shorter than the base32 form, but the minor usability improvement is not worth the documentation and specification confusion of using a non-standard encoding. So we stick with base32. + +Ported to Python 3. ''' +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + + +from future.utils import PY2 +if PY2: + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 import six diff --git a/src/allmydata/crypto/error.py b/src/allmydata/crypto/error.py index 62c0b3e5b..b105805a5 100644 --- a/src/allmydata/crypto/error.py +++ b/src/allmydata/crypto/error.py @@ -1,6 +1,16 @@ """ Exceptions raise by allmydata.crypto.* modules + +Ported to Python 3. """ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 class BadSignature(Exception): diff --git a/src/allmydata/crypto/rsa.py b/src/allmydata/crypto/rsa.py index e82bf12d1..95c39bcc6 100644 --- a/src/allmydata/crypto/rsa.py +++ b/src/allmydata/crypto/rsa.py @@ -9,8 +9,17 @@ features of any objects that `cryptography` documents. That is, the public and private keys are opaque objects; DO NOT depend on any of their methods. -""" +Ported to Python 3. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 from cryptography.exceptions import InvalidSignature from cryptography.hazmat.backends import default_backend diff --git a/src/allmydata/crypto/util.py b/src/allmydata/crypto/util.py index 6aa1f0973..6c6ed1059 100644 --- a/src/allmydata/crypto/util.py +++ b/src/allmydata/crypto/util.py @@ -1,6 +1,16 @@ """ Utilities used by allmydata.crypto modules + +Ported to Python 3. """ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 from allmydata.crypto.error import BadPrefixError diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index f96a6f1e4..c29184b6c 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -15,6 +15,12 @@ if PY2: # Keep these sorted alphabetically, to reduce merge conflicts: PORTED_MODULES = [ + "allmydata.crypto", + "allmydata.crypto.aes", + "allmydata.crypto.ed25519", + "allmydata.crypto.error", + "allmydata.crypto.rsa", + "allmydata.crypto.util", "allmydata.hashtree", "allmydata.util.abbreviate", "allmydata.util.assertutil", From c0cabeaca0277ccf36e5fea902ebf5a95f5599bf Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 3 Aug 2020 11:08:16 -0400 Subject: [PATCH 13/79] Update ratchet. --- misc/python3/ratchet-passing | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/misc/python3/ratchet-passing b/misc/python3/ratchet-passing index b9beb360e..d214396ff 100644 --- a/misc/python3/ratchet-passing +++ b/misc/python3/ratchet-passing @@ -24,6 +24,33 @@ allmydata.test.test_base62.Base62.test_known_values allmydata.test.test_base62.Base62.test_num_octets_that_encode_to_this_many_chars allmydata.test.test_base62.Base62.test_odd_sizes allmydata.test.test_base62.Base62.test_roundtrip +allmydata.test.test_crypto.TestEd25519.test_deserialize_private_not_bytes +allmydata.test.test_crypto.TestEd25519.test_deserialize_public_not_bytes +allmydata.test.test_crypto.TestEd25519.test_key_serialization +allmydata.test.test_crypto.TestEd25519.test_sign_invalid_pubkey +allmydata.test.test_crypto.TestEd25519.test_signature_data_not_bytes +allmydata.test.test_crypto.TestEd25519.test_signature_not_bytes +allmydata.test.test_crypto.TestEd25519.test_signed_data_not_bytes +allmydata.test.test_crypto.TestEd25519.test_verify_invalid_pubkey +allmydata.test.test_crypto.TestRegression.test_aes_no_iv_process_long_input +allmydata.test.test_crypto.TestRegression.test_aes_no_iv_process_short_input +allmydata.test.test_crypto.TestRegression.test_aes_with_iv_process_long_input +allmydata.test.test_crypto.TestRegression.test_aes_with_iv_process_short_input +allmydata.test.test_crypto.TestRegression.test_decode_ed15519_keypair +allmydata.test.test_crypto.TestRegression.test_decode_rsa_keypair +allmydata.test.test_crypto.TestRegression.test_encrypt_data_not_bytes +allmydata.test.test_crypto.TestRegression.test_incorrect_iv_size +allmydata.test.test_crypto.TestRegression.test_iv_not_bytes +allmydata.test.test_crypto.TestRegression.test_key_incorrect_size +allmydata.test.test_crypto.TestRegression.test_old_start_up_test +allmydata.test.test_crypto.TestRsa.test_keys +allmydata.test.test_crypto.TestRsa.test_sign_invalid_pubkey +allmydata.test.test_crypto.TestRsa.test_verify_invalid_pubkey +allmydata.test.test_crypto.TestUtil.test_remove_prefix_bad +allmydata.test.test_crypto.TestUtil.test_remove_prefix_entire_string +allmydata.test.test_crypto.TestUtil.test_remove_prefix_good +allmydata.test.test_crypto.TestUtil.test_remove_prefix_partial +allmydata.test.test_crypto.TestUtil.test_remove_prefix_zero allmydata.test.test_deferredutil.DeferredUtilTests.test_failure allmydata.test.test_deferredutil.DeferredUtilTests.test_gather_results allmydata.test.test_deferredutil.DeferredUtilTests.test_success From bfbca87f265d00241cd9892ac5fb5de19c3c4ce7 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 3 Aug 2020 11:31:02 -0400 Subject: [PATCH 14/79] Missing from previous PR. --- src/allmydata/hashtree.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/allmydata/hashtree.py b/src/allmydata/hashtree.py index 77798b3c2..c692e3597 100644 --- a/src/allmydata/hashtree.py +++ b/src/allmydata/hashtree.py @@ -45,6 +45,8 @@ Written by Connelly Barnes in 2005 and released into the public domain with no warranty of any kind, either expressed or implied. It probably won't make your computer catch on fire, or eat your children, but it might. Use at your own risk. + +Ported to Python 3. """ from __future__ import absolute_import From 1703230a2a5bc48278581e558e9ca9091d7d5647 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 3 Aug 2020 11:33:21 -0400 Subject: [PATCH 15/79] Use nummedobj from pyutil. --- src/allmydata/util/log.py | 2 +- src/allmydata/util/nummedobj.py | 42 --------------------------------- 2 files changed, 1 insertion(+), 43 deletions(-) delete mode 100644 src/allmydata/util/nummedobj.py diff --git a/src/allmydata/util/log.py b/src/allmydata/util/log.py index 454002000..8ebf1262b 100644 --- a/src/allmydata/util/log.py +++ b/src/allmydata/util/log.py @@ -1,4 +1,4 @@ -from allmydata.util import nummedobj +from pyutil import nummedobj from foolscap.logging import log from twisted.python import log as tw_log diff --git a/src/allmydata/util/nummedobj.py b/src/allmydata/util/nummedobj.py deleted file mode 100644 index 50d7c6454..000000000 --- a/src/allmydata/util/nummedobj.py +++ /dev/null @@ -1,42 +0,0 @@ -import collections, itertools, functools - -objnums = collections.defaultdict(itertools.count) - - -@functools.total_ordering -class NummedObj(object): - """ - This is useful for nicer debug printouts. Instead of objects of the same class being - distinguished from one another by their memory address, they each get a unique number, which - can be read as "the first object of this class", "the second object of this class", etc. This - is especially useful because separate runs of a program will yield identical debug output, - (assuming that the objects get created in the same order in each run). This makes it possible - to diff outputs from separate runs to see what changed, without having to ignore a difference - on every line due to different memory addresses of objects. - """ - - def __init__(self, klass=None): - """ - @param klass: in which class are you counted? If default value of `None', then self.__class__ will be used. - """ - if klass is None: - klass = self.__class__ - self._classname = klass.__name__ - - self._objid = objnums[self._classname].next() - - def __repr__(self): - return "<%s #%d>" % (self._classname, self._objid,) - - def __lt__(self, other): - if isinstance(other, NummedObj): - return (self._objid, self._classname,) < (other._objid, other._classname,) - return NotImplemented - - def __eq__(self, other): - if isinstance(other, NummedObj): - return (self._objid, self._classname,) == (other._objid, other._classname,) - return NotImplemented - - def __hash__(self): - return id(self) From 02daa12031105f36649828b5e024bbb27ed5d295 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 3 Aug 2020 11:33:29 -0400 Subject: [PATCH 16/79] Move logging tests into their own, Python 3-compatible, module. --- src/allmydata/test/test_util.py | 16 ---------------- src/allmydata/util/_python3.py | 1 + 2 files changed, 1 insertion(+), 16 deletions(-) diff --git a/src/allmydata/test/test_util.py b/src/allmydata/test/test_util.py index 1e9dd0f49..bd0e16df1 100644 --- a/src/allmydata/test/test_util.py +++ b/src/allmydata/test/test_util.py @@ -6,13 +6,11 @@ import yaml from twisted.trial import unittest from twisted.internet import defer, reactor -from twisted.python.failure import Failure from allmydata.util import idlib, mathutil from allmydata.util import fileutil from allmydata.util import limiter, pollmixin from allmydata.util import yamlutil -from allmydata.util import log as tahoe_log from allmydata.util.fileutil import EncryptedTemporaryFile from allmydata.test.common_util import ReallyEqualMixin @@ -528,20 +526,6 @@ class EqButNotIs(object): return self.x == other -class SampleError(Exception): - pass - -class Log(unittest.TestCase): - def test_err(self): - try: - raise SampleError("simple sample") - except: - f = Failure() - tahoe_log.err(format="intentional sample error", - failure=f, level=tahoe_log.OPERATIONAL, umid="wO9UoQ") - self.flushLoggedErrors(SampleError) - - class YAML(unittest.TestCase): def test_convert(self): data = yaml.safe_dump(["str", u"unicode", u"\u1234nicode"]) diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 8a75a8fc5..046468eeb 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -48,6 +48,7 @@ PORTED_TEST_MODULES = [ "allmydata.test.test_hashutil", "allmydata.test.test_humanreadable", "allmydata.test.test_iputil", + "allmydata.test.test_log", "allmydata.test.test_netstring", "allmydata.test.test_observer", "allmydata.test.test_pipeline", From 622ed2f971a94f54de19f2cdb1ee2459723155a4 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 3 Aug 2020 14:09:28 -0400 Subject: [PATCH 17/79] Standalone logging tests, most of them new, and an attempt to fix the parent msgid logic so it's a little less broken. --- src/allmydata/test/test_log.py | 113 +++++++++++++++++++++++++++++++++ src/allmydata/util/log.py | 4 +- 2 files changed, 115 insertions(+), 2 deletions(-) create mode 100644 src/allmydata/test/test_log.py diff --git a/src/allmydata/test/test_log.py b/src/allmydata/test/test_log.py new file mode 100644 index 000000000..00ba03013 --- /dev/null +++ b/src/allmydata/test/test_log.py @@ -0,0 +1,113 @@ +""" +Tests for allmydata.util.log. + +Ported to Python 3. +""" + +from __future__ import unicode_literals +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from future.utils import PY2 +if PY2: + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + +from twisted.trial import unittest +from twisted.python.failure import Failure + +from foolscap.logging import log + +from allmydata.util import log as tahoe_log + + +class SampleError(Exception): + pass + + +class LoggingObject0(tahoe_log.PrefixingLogMixin): + pass + + +class LoggingObject1(tahoe_log.PrefixingLogMixin): + pass + + +class LoggingObject2(tahoe_log.PrefixingLogMixin): + pass + + +class LoggingObject3(tahoe_log.PrefixingLogMixin): + pass + + +class Log(unittest.TestCase): + def setUp(self): + self.messages = [] + + def msg(msg, facility, parent, *args, **kwargs): + self.messages.append((msg, facility, parent, args, kwargs)) + return "msg{}".format(len(self.messages)) + + self.patch(log, "msg", msg) + + def test_err(self): + """Logging with log.err() causes tests to fail.""" + try: + raise SampleError("simple sample") + except: + f = Failure() + tahoe_log.err(format="intentional sample error", + failure=f, level=tahoe_log.OPERATIONAL, umid="wO9UoQ") + self.flushLoggedErrors(SampleError) + + def test_default_facility(self): + obj = LoggingObject1(facility="defaultfac") + obj.log("hello") + obj.log("world", facility="override") + self.assertEqual(self.messages[-2][1], "defaultfac") + self.assertEqual(self.messages[-1][1], "override") + + def test_with_prefix(self): + obj = LoggingObject0("fac", prefix="pre1") + obj.log("hello") + obj.log("world") + self.assertEqual(self.messages[-2][0], '(pre1): hello') + self.assertEqual(self.messages[-1][0], '(pre1): world') + + def test_no_prefix(self): + obj = LoggingObject2() + obj.log("hello") + obj.log("world") + self.assertEqual(self.messages[-2][0], ': hello') + self.assertEqual(self.messages[-1][0], ': world') + + def test_numming(self): + obj = LoggingObject3() + obj2 = LoggingObject3() + obj.log("hello") + obj2.log("world") + self.assertEqual(self.messages[-2][0], ': hello') + self.assertEqual(self.messages[-1][0], ': world') + + def test_parent_id(self): + obj = LoggingObject1() + result = obj.log("zero") + self.assertEqual(result, "msg1") + obj.log("one", parent="par1") + obj.log("two", parent="par2") + obj.log("three") + obj.log("four") + self.assertEqual([m[2] for m in self.messages], + [None, "par1", "par2", "msg1", "msg1"]) + + def test_grandparent_id(self): + obj = LoggingObject1(grandparentmsgid="grand") + result = obj.log("zero") + self.assertEqual(result, "msg1") + obj.log("one", parent="par1") + obj.log("two", parent="par2") + obj.log("three") + obj.log("four") + self.assertEqual([m[2] for m in self.messages], + ["grand", "par1", "par2", "msg1", "msg1"]) diff --git a/src/allmydata/util/log.py b/src/allmydata/util/log.py index 8ebf1262b..c1966aadb 100644 --- a/src/allmydata/util/log.py +++ b/src/allmydata/util/log.py @@ -36,8 +36,8 @@ class LogMixin(object): def log(self, msg, facility=None, parent=None, *args, **kwargs): if facility is None: facility = self._facility - pmsgid = None - if parent is None: + pmsgid = parent + if pmsgid is None: pmsgid = self._parentmsgid if pmsgid is None: pmsgid = self._grandparentmsgid From 2a623e0b05e0162d9332eea40429b4115c5984af Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 3 Aug 2020 14:13:16 -0400 Subject: [PATCH 18/79] Port to Python 3. --- misc/python3/ratchet-passing | 7 +++++++ newsfragments/3365.minor | 0 src/allmydata/util/_python3.py | 1 + src/allmydata/util/log.py | 14 ++++++++++++++ 4 files changed, 22 insertions(+) create mode 100644 newsfragments/3365.minor diff --git a/misc/python3/ratchet-passing b/misc/python3/ratchet-passing index b9beb360e..2edba65c0 100644 --- a/misc/python3/ratchet-passing +++ b/misc/python3/ratchet-passing @@ -54,6 +54,13 @@ allmydata.test.test_iputil.ListAddresses.test_list_async_mock_ip_addr allmydata.test.test_iputil.ListAddresses.test_list_async_mock_route allmydata.test.test_iputil.ListenOnUsed.test_random_port allmydata.test.test_iputil.ListenOnUsed.test_specific_port +allmydata.test.test_log.Log.test_default_facility +allmydata.test.test_log.Log.test_err +allmydata.test.test_log.Log.test_grandparent_id +allmydata.test.test_log.Log.test_no_prefix +allmydata.test.test_log.Log.test_numming +allmydata.test.test_log.Log.test_parent_id +allmydata.test.test_log.Log.test_with_prefix allmydata.test.test_netstring.Netstring.test_encode allmydata.test.test_netstring.Netstring.test_extra allmydata.test.test_netstring.Netstring.test_nested diff --git a/newsfragments/3365.minor b/newsfragments/3365.minor new file mode 100644 index 000000000..e69de29bb diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 046468eeb..a1c727697 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -25,6 +25,7 @@ PORTED_MODULES = [ "allmydata.util.hashutil", "allmydata.util.humanreadable", "allmydata.util.iputil", + "allmydata.util.log", "allmydata.util.mathutil", "allmydata.util.namespace", "allmydata.util.netstring", diff --git a/src/allmydata/util/log.py b/src/allmydata/util/log.py index c1966aadb..cd20c764d 100644 --- a/src/allmydata/util/log.py +++ b/src/allmydata/util/log.py @@ -1,3 +1,17 @@ +""" +Logging utilities. + +Ported to Python 3. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from pyutil import nummedobj from foolscap.logging import log From a2f78ca7034c8cc231c7cc9f1c39c1fa6859233b Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 17 Jul 2020 16:26:49 -0400 Subject: [PATCH 19/79] don't have a default --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index dd423e2af..04b208e34 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -450,7 +450,7 @@ jobs: docker \ build \ --build-arg TAG=${TAG} \ - --build-arg PYTHON_VERSION=${PYTHON_VERSION:-python2} + --build-arg PYTHON_VERSION=${PYTHON_VERSION} -t tahoelafsci/${DISTRO}:${TAG} \ -f ~/project/.circleci/Dockerfile.${DISTRO} \ ~/project/ From 95050fefd6f5e66ef17c98ef98f7d02b1848196c Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 17 Jul 2020 16:26:56 -0400 Subject: [PATCH 20/79] do have line continuation --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 04b208e34..dcaf6cf14 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -450,7 +450,7 @@ jobs: docker \ build \ --build-arg TAG=${TAG} \ - --build-arg PYTHON_VERSION=${PYTHON_VERSION} + --build-arg PYTHON_VERSION=${PYTHON_VERSION} \ -t tahoelafsci/${DISTRO}:${TAG} \ -f ~/project/.circleci/Dockerfile.${DISTRO} \ ~/project/ From 704f751b06204ff7794771cb71c0b62950a7a13d Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 17 Jul 2020 16:45:10 -0400 Subject: [PATCH 21/79] untabify --- .circleci/Dockerfile.debian | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.circleci/Dockerfile.debian b/.circleci/Dockerfile.debian index 394495e17..87d5487d6 100644 --- a/.circleci/Dockerfile.debian +++ b/.circleci/Dockerfile.debian @@ -9,16 +9,16 @@ ENV BUILD_SRC_ROOT /tmp/project RUN apt-get --quiet update && \ apt-get --quiet --yes install \ - git \ - lsb-release \ + git \ + lsb-release \ sudo \ - build-essential \ - python${PYTHON_VERSION} \ - python${PYTHON_VERSION}-dev \ - libffi-dev \ - libssl-dev \ - libyaml-dev \ - virtualenv + build-essential \ + python${PYTHON_VERSION} \ + python${PYTHON_VERSION}-dev \ + libffi-dev \ + libssl-dev \ + libyaml-dev \ + virtualenv # Get the project source. This is better than it seems. CircleCI will # *update* this checkout on each job run, saving us more time per-job. From ee32b3555ac8ee7880709e26e65dfa47a1737cb4 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 17 Jul 2020 20:00:13 -0400 Subject: [PATCH 22/79] Burn the Python version into the Docker image tag --- .circleci/config.yml | 38 ++++++++++++++++++++++---------------- 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index dcaf6cf14..77b661ab9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -121,7 +121,7 @@ jobs: debian-9: &DEBIAN docker: - - image: "tahoelafsci/debian:9" + - image: "tahoelafsci/debian:9-py2" user: "nobody" environment: &UTF_8_ENVIRONMENT @@ -198,14 +198,14 @@ jobs: debian-8: <<: *DEBIAN docker: - - image: "tahoelafsci/debian:8" + - image: "tahoelafsci/debian:8-py2" user: "nobody" pypy2.7-buster: <<: *DEBIAN docker: - - image: "tahoelafsci/pypy:2.7-buster" + - image: "tahoelafsci/pypy:buster-py2" user: "nobody" environment: @@ -261,19 +261,23 @@ jobs: ubuntu-16.04: <<: *DEBIAN docker: - - image: "tahoelafsci/ubuntu:16.04" + - image: "tahoelafsci/ubuntu:16.04-py2" user: "nobody" ubuntu-18.04: &UBUNTU_18_04 <<: *DEBIAN docker: - - image: "tahoelafsci/ubuntu:18.04" + - image: "tahoelafsci/ubuntu:18.04-py2" user: "nobody" python3.6: <<: *UBUNTU_18_04 + docker: + - image: "tahoelafsci/ubuntu:18.04-py3" + user: "nobody" + environment: <<: *UTF_8_ENVIRONMENT TAHOE_LAFS_TOX_ENVIRONMENT: "py36" @@ -281,7 +285,7 @@ jobs: centos-8: &RHEL_DERIV docker: - - image: "tahoelafsci/centos:8" + - image: "tahoelafsci/centos:8-py2" user: "nobody" environment: *UTF_8_ENVIRONMENT @@ -303,20 +307,20 @@ jobs: fedora-28: <<: *RHEL_DERIV docker: - - image: "tahoelafsci/fedora:28" + - image: "tahoelafsci/fedora:28-py2" user: "nobody" fedora-29: <<: *RHEL_DERIV docker: - - image: "tahoelafsci/fedora:29" + - image: "tahoelafsci/fedora:29-py2" user: "nobody" slackware-14.2: docker: - - image: "tahoelafsci/slackware:14.2" + - image: "tahoelafsci/slackware:14.2-py2" user: "nobody" environment: *UTF_8_ENVIRONMENT @@ -397,8 +401,9 @@ jobs: - image: "docker:17.05.0-ce-git" environment: - DISTRO: "tahoelafsci/:foo" - TAG: "tahoelafsci/distro:" + DISTRO: "tahoelafsci/:foo-py2" + TAG: "tahoelafsci/distro:-py2" + PYTHON_VERSION: "tahoelafsci/distro:tag-py Date: Fri, 17 Jul 2020 20:36:17 -0400 Subject: [PATCH 23/79] FROM creates a new build stage which discards all prior ARGs --- .circleci/Dockerfile.centos | 2 +- .circleci/Dockerfile.debian | 2 +- .circleci/Dockerfile.fedora | 2 +- .circleci/Dockerfile.slackware | 2 +- .circleci/Dockerfile.ubuntu | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.circleci/Dockerfile.centos b/.circleci/Dockerfile.centos index 632e13137..9070d71d9 100644 --- a/.circleci/Dockerfile.centos +++ b/.circleci/Dockerfile.centos @@ -1,6 +1,6 @@ ARG TAG -ARG PYTHON_VERSION FROM centos:${TAG} +ARG PYTHON_VERSION ENV WHEELHOUSE_PATH /tmp/wheelhouse ENV VIRTUALENV_PATH /tmp/venv diff --git a/.circleci/Dockerfile.debian b/.circleci/Dockerfile.debian index 87d5487d6..96c54736c 100644 --- a/.circleci/Dockerfile.debian +++ b/.circleci/Dockerfile.debian @@ -1,6 +1,6 @@ ARG TAG -ARG PYTHON_VERSION FROM debian:${TAG} +ARG PYTHON_VERSION ENV WHEELHOUSE_PATH /tmp/wheelhouse ENV VIRTUALENV_PATH /tmp/venv diff --git a/.circleci/Dockerfile.fedora b/.circleci/Dockerfile.fedora index f17140f3e..e60dbb85d 100644 --- a/.circleci/Dockerfile.fedora +++ b/.circleci/Dockerfile.fedora @@ -1,6 +1,6 @@ ARG TAG -ARG PYTHON_VERSION FROM fedora:${TAG} +ARG PYTHON_VERSION ENV WHEELHOUSE_PATH /tmp/wheelhouse ENV VIRTUALENV_PATH /tmp/venv diff --git a/.circleci/Dockerfile.slackware b/.circleci/Dockerfile.slackware index 97ee07609..4b6d96f95 100644 --- a/.circleci/Dockerfile.slackware +++ b/.circleci/Dockerfile.slackware @@ -1,6 +1,6 @@ ARG TAG -ARG PYTHON_VERSION FROM vbatts/slackware:${TAG} +ARG PYTHON_VERSION ENV WHEELHOUSE_PATH /tmp/wheelhouse ENV VIRTUALENV_PATH /tmp/venv diff --git a/.circleci/Dockerfile.ubuntu b/.circleci/Dockerfile.ubuntu index 4ca2319a9..2fcc60f5a 100644 --- a/.circleci/Dockerfile.ubuntu +++ b/.circleci/Dockerfile.ubuntu @@ -1,6 +1,6 @@ ARG TAG -ARG PYTHON_VERSION FROM ubuntu:${TAG} +ARG PYTHON_VERSION ENV WHEELHOUSE_PATH /tmp/wheelhouse ENV VIRTUALENV_PATH /tmp/venv From b99ec195cdb86d51a8d1fbb532d289715c735269 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 3 Aug 2020 16:29:08 -0400 Subject: [PATCH 24/79] See if this helps the py36 zfec installation --- .circleci/config.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 77b661ab9..6dbdf8ff4 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -281,6 +281,8 @@ jobs: environment: <<: *UTF_8_ENVIRONMENT TAHOE_LAFS_TOX_ENVIRONMENT: "py36" + # zfec 1.5.3 is uninstallable without a utf-8 locale + LC_ALL: "en_US.utf-8" centos-8: &RHEL_DERIV From ec7531f1138d3ccedb07027afb9592f510d73b2c Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 3 Aug 2020 16:40:04 -0400 Subject: [PATCH 25/79] maybe this will let it get to the right place --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index ab467abd0..205fe1531 100644 --- a/tox.ini +++ b/tox.ini @@ -50,7 +50,7 @@ commands = [testenv:py36] # git inside of ratchet.sh needs $HOME. -passenv = HOME +passenv = HOME LC_ALL commands = {toxinidir}/misc/python3/ratchet.sh [testenv:integration] From 8348e7032b24b01dda7270f73613a792f0a15fcc Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 3 Aug 2020 16:53:27 -0400 Subject: [PATCH 26/79] this is basically already set (LANG in UTF_8_ENVIRONMENT) --- .circleci/config.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 6dbdf8ff4..77b661ab9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -281,8 +281,6 @@ jobs: environment: <<: *UTF_8_ENVIRONMENT TAHOE_LAFS_TOX_ENVIRONMENT: "py36" - # zfec 1.5.3 is uninstallable without a utf-8 locale - LC_ALL: "en_US.utf-8" centos-8: &RHEL_DERIV From a513c7ab28573b73bc65b56a573b9588b2d99ffc Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 3 Aug 2020 16:53:40 -0400 Subject: [PATCH 27/79] try setting it in here --- tox.ini | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 205fe1531..9a11df98c 100644 --- a/tox.ini +++ b/tox.ini @@ -50,7 +50,9 @@ commands = [testenv:py36] # git inside of ratchet.sh needs $HOME. -passenv = HOME LC_ALL +passenv = HOME +setenv = + LANG=en_US.UTF-8 commands = {toxinidir}/misc/python3/ratchet.sh [testenv:integration] From 612afdcf7c572a509227f98bb276fcd1f9b07404 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 3 Aug 2020 16:58:25 -0400 Subject: [PATCH 28/79] that is useless, the error is in the docker image build --- tox.ini | 2 -- 1 file changed, 2 deletions(-) diff --git a/tox.ini b/tox.ini index 9a11df98c..ab467abd0 100644 --- a/tox.ini +++ b/tox.ini @@ -51,8 +51,6 @@ commands = [testenv:py36] # git inside of ratchet.sh needs $HOME. passenv = HOME -setenv = - LANG=en_US.UTF-8 commands = {toxinidir}/misc/python3/ratchet.sh [testenv:integration] From ab9ccae1d19ca76bbd9f109f11533546693581c0 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 3 Aug 2020 16:58:36 -0400 Subject: [PATCH 29/79] try doing the `pip wheel` with the right LANG --- .circleci/populate-wheelhouse.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.circleci/populate-wheelhouse.sh b/.circleci/populate-wheelhouse.sh index 75afb6f6f..519a80cac 100755 --- a/.circleci/populate-wheelhouse.sh +++ b/.circleci/populate-wheelhouse.sh @@ -36,8 +36,9 @@ PIP="${BOOTSTRAP_VENV}/bin/pip" # Tell pip where it can find any existing wheels. export PIP_FIND_LINKS="file://${WHEELHOUSE_PATH}" -# Populate the wheelhouse, if necessary. -"${PIP}" \ +# Populate the wheelhouse, if necessary. zfec 1.5.3 can only be built with a +# UTF-8 environment so make sure we have one, at least for this invocation. +LANG="en_US.UTF-8" "${PIP}" \ wheel \ --wheel-dir "${WHEELHOUSE_PATH}" \ "${PROJECT_ROOT}"[test] \ From 1b0f83599d196c84da2f4ad9961ee767e2ae4ff8 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 3 Aug 2020 17:08:45 -0400 Subject: [PATCH 30/79] See if we can get the image names right --- .circleci/config.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 77b661ab9..203993f4e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -121,7 +121,7 @@ jobs: debian-9: &DEBIAN docker: - - image: "tahoelafsci/debian:9-py2" + - image: "tahoelafsci/debian:9-py2.7" user: "nobody" environment: &UTF_8_ENVIRONMENT @@ -198,7 +198,7 @@ jobs: debian-8: <<: *DEBIAN docker: - - image: "tahoelafsci/debian:8-py2" + - image: "tahoelafsci/debian:8-py2.7" user: "nobody" @@ -261,14 +261,14 @@ jobs: ubuntu-16.04: <<: *DEBIAN docker: - - image: "tahoelafsci/ubuntu:16.04-py2" + - image: "tahoelafsci/ubuntu:16.04-py2.7" user: "nobody" ubuntu-18.04: &UBUNTU_18_04 <<: *DEBIAN docker: - - image: "tahoelafsci/ubuntu:18.04-py2" + - image: "tahoelafsci/ubuntu:18.04-py2.7" user: "nobody" @@ -307,7 +307,7 @@ jobs: fedora-28: <<: *RHEL_DERIV docker: - - image: "tahoelafsci/fedora:28-py2" + - image: "tahoelafsci/fedora:28-py" user: "nobody" From a47bd93fd045551b7a716eb284987862b917c5ff Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 3 Aug 2020 17:12:15 -0400 Subject: [PATCH 31/79] news fragment --- newsfragments/3336.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3336.minor diff --git a/newsfragments/3336.minor b/newsfragments/3336.minor new file mode 100644 index 000000000..e69de29bb From f0ddda966533f10ca341a53e0efc0435dcac4edb Mon Sep 17 00:00:00 2001 From: Chad Whitacre Date: Fri, 31 Jul 2020 08:14:39 -0400 Subject: [PATCH 32/79] Add a codecheck for tabs in indentation --- newsfragments/3326.minor | 0 setup.cfg | 8 +++++--- 2 files changed, 5 insertions(+), 3 deletions(-) create mode 100644 newsfragments/3326.minor diff --git a/newsfragments/3326.minor b/newsfragments/3326.minor new file mode 100644 index 000000000..e69de29bb diff --git a/setup.cfg b/setup.cfg index c2d224225..f4539279e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -7,6 +7,8 @@ bdist_egg = update_version bdist_egg bdist_wheel = update_version bdist_wheel [flake8] -# For now, only use pyflakes errors; flake8 is still helpful because it allows -# ignoring specific errors/warnings when needed. -select = F \ No newline at end of file +# Enforce all pyflakes constraints, and also prohibit tabs for indentation. +# Reference: +# https://flake8.pycqa.org/en/latest/user/error-codes.html +# https://pycodestyle.pycqa.org/en/latest/intro.html#error-codes +select = F, W191 From 2a6cd8eb58f273c721725bb8eeb265fb6425a472 Mon Sep 17 00:00:00 2001 From: Chad Whitacre Date: Fri, 31 Jul 2020 07:41:33 -0400 Subject: [PATCH 33/79] Get `tox -e codechecks` to work (on macOS?) Without this, git fails underneath towncrier with an "error: Could not expand include path '~/.gitcinclude'". See: https://stackoverflow.com/q/36908041 I had added similar for `tox -e py36` in d25c8b1a. --- newsfragments/3366.minor | 0 tox.ini | 4 +++- 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 newsfragments/3366.minor diff --git a/newsfragments/3366.minor b/newsfragments/3366.minor new file mode 100644 index 000000000..e69de29bb diff --git a/tox.ini b/tox.ini index ab467abd0..4356cfd24 100644 --- a/tox.ini +++ b/tox.ini @@ -49,7 +49,7 @@ commands = tahoe --version [testenv:py36] -# git inside of ratchet.sh needs $HOME. +# On macOS, git inside of ratchet.sh needs $HOME. passenv = HOME commands = {toxinidir}/misc/python3/ratchet.sh @@ -77,6 +77,8 @@ commands = coverage xml [testenv:codechecks] +# On macOS, git inside of towncrier needs $HOME. +passenv = HOME whitelist_externals = /bin/mv commands = From ce33101d080c21e62653b12385480ccc894b2177 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 4 Aug 2020 15:03:06 -0400 Subject: [PATCH 34/79] Remove unused import. --- src/allmydata/test/test_crypto.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/allmydata/test/test_crypto.py b/src/allmydata/test/test_crypto.py index 7f6271ef3..498723cf4 100644 --- a/src/allmydata/test/test_crypto.py +++ b/src/allmydata/test/test_crypto.py @@ -9,7 +9,6 @@ if PY2: from future.utils import native_bytes -import six import unittest from base64 import b64decode From ff7cf4d731242744c2b971f485b48e50a9cce02f Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 4 Aug 2020 15:15:32 -0400 Subject: [PATCH 35/79] Test improvements. --- src/allmydata/test/test_log.py | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/src/allmydata/test/test_log.py b/src/allmydata/test/test_log.py index 00ba03013..1d24d7f7b 100644 --- a/src/allmydata/test/test_log.py +++ b/src/allmydata/test/test_log.py @@ -59,9 +59,14 @@ class Log(unittest.TestCase): f = Failure() tahoe_log.err(format="intentional sample error", failure=f, level=tahoe_log.OPERATIONAL, umid="wO9UoQ") - self.flushLoggedErrors(SampleError) + result = self.flushLoggedErrors(SampleError) + self.assertEqual(len(result), 1) def test_default_facility(self): + """ + If facility is passed to PrefixingLogMixin.__init__, it is used as + default facility. + """ obj = LoggingObject1(facility="defaultfac") obj.log("hello") obj.log("world", facility="override") @@ -69,6 +74,10 @@ class Log(unittest.TestCase): self.assertEqual(self.messages[-1][1], "override") def test_with_prefix(self): + """ + If prefix is passed to PrefixingLogMixin.__init__, it is used in + message rendering. + """ obj = LoggingObject0("fac", prefix="pre1") obj.log("hello") obj.log("world") @@ -76,6 +85,10 @@ class Log(unittest.TestCase): self.assertEqual(self.messages[-1][0], '(pre1): world') def test_no_prefix(self): + """ + If no prefix is passed to PrefixingLogMixin.__init__, it is not used in + message rendering. + """ obj = LoggingObject2() obj.log("hello") obj.log("world") @@ -83,6 +96,10 @@ class Log(unittest.TestCase): self.assertEqual(self.messages[-1][0], ': world') def test_numming(self): + """ + Objects inheriting from PrefixingLogMixin get a unique number from a + class-specific counter. + """ obj = LoggingObject3() obj2 = LoggingObject3() obj.log("hello") @@ -91,6 +108,12 @@ class Log(unittest.TestCase): self.assertEqual(self.messages[-1][0], ': world') def test_parent_id(self): + """ + The parent message id can be passed in, otherwise the first message's + id is used as the parent. + + This logic is pretty bogus, but that's what the code does. + """ obj = LoggingObject1() result = obj.log("zero") self.assertEqual(result, "msg1") @@ -102,6 +125,10 @@ class Log(unittest.TestCase): [None, "par1", "par2", "msg1", "msg1"]) def test_grandparent_id(self): + """ + If grandparent message id is given, it's used as parent id of the first + message. + """ obj = LoggingObject1(grandparentmsgid="grand") result = obj.log("zero") self.assertEqual(result, "msg1") From 0bef1eb4e2c6f76c82df67cc6d990e9f8f285b0d Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 5 Aug 2020 11:25:40 -0400 Subject: [PATCH 36/79] Support prefix that is bytes. --- src/allmydata/test/test_log.py | 11 +++++++++++ src/allmydata/util/log.py | 2 ++ 2 files changed, 13 insertions(+) diff --git a/src/allmydata/test/test_log.py b/src/allmydata/test/test_log.py index 1d24d7f7b..2c26e8ff4 100644 --- a/src/allmydata/test/test_log.py +++ b/src/allmydata/test/test_log.py @@ -84,6 +84,17 @@ class Log(unittest.TestCase): self.assertEqual(self.messages[-2][0], '(pre1): hello') self.assertEqual(self.messages[-1][0], '(pre1): world') + def test_with_bytes_prefix(self): + """ + If bytes prefix is passed to PrefixingLogMixin.__init__, it is used in + message rendering. + """ + obj = LoggingObject0("fac", prefix=b"pre1") + obj.log("hello") + obj.log("world") + self.assertEqual(self.messages[-2][0], '(pre1): hello') + self.assertEqual(self.messages[-1][0], '(pre1): world') + def test_no_prefix(self): """ If no prefix is passed to PrefixingLogMixin.__init__, it is not used in diff --git a/src/allmydata/util/log.py b/src/allmydata/util/log.py index cd20c764d..dcb5838a3 100644 --- a/src/allmydata/util/log.py +++ b/src/allmydata/util/log.py @@ -68,6 +68,8 @@ class PrefixingLogMixin(nummedobj.NummedObj, LogMixin): LogMixin.__init__(self, facility, grandparentmsgid) if prefix: + if isinstance(prefix, bytes): + prefix = prefix.decode("utf-8", errors="replace") self._prefix = "%s(%s): " % (self.__repr__(), prefix) else: self._prefix = "%s: " % (self.__repr__(),) From 70851fe75384ab3d7015480329ab6995372b7896 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 5 Aug 2020 11:53:23 -0400 Subject: [PATCH 37/79] Death to newint. --- src/allmydata/crypto/__init__.py | 2 +- src/allmydata/crypto/aes.py | 2 +- src/allmydata/crypto/ed25519.py | 2 +- src/allmydata/crypto/error.py | 2 +- src/allmydata/crypto/rsa.py | 2 +- src/allmydata/crypto/util.py | 2 +- src/allmydata/hashtree.py | 2 +- src/allmydata/test/common_py3.py | 2 +- src/allmydata/test/test_abbreviate.py | 2 +- src/allmydata/test/test_base32.py | 2 +- src/allmydata/test/test_base62.py | 2 +- src/allmydata/test/test_crypto.py | 2 +- src/allmydata/test/test_deferredutil.py | 2 +- src/allmydata/test/test_dictutil.py | 2 +- src/allmydata/test/test_hashtree.py | 2 +- src/allmydata/test/test_hashutil.py | 2 +- src/allmydata/test/test_humanreadable.py | 2 +- src/allmydata/test/test_iputil.py | 2 +- src/allmydata/test/test_netstring.py | 2 +- src/allmydata/test/test_observer.py | 2 +- src/allmydata/test/test_pipeline.py | 2 +- src/allmydata/test/test_python3.py | 2 +- src/allmydata/test/test_spans.py | 2 +- src/allmydata/test/test_statistics.py | 2 +- src/allmydata/test/test_time_format.py | 2 +- src/allmydata/test/test_version.py | 2 +- src/allmydata/util/_python3.py | 2 +- src/allmydata/util/abbreviate.py | 2 +- src/allmydata/util/assertutil.py | 2 +- src/allmydata/util/base32.py | 2 +- src/allmydata/util/base62.py | 2 +- src/allmydata/util/deferredutil.py | 2 +- src/allmydata/util/gcutil.py | 2 +- src/allmydata/util/hashutil.py | 2 +- src/allmydata/util/humanreadable.py | 2 +- src/allmydata/util/iputil.py | 2 +- src/allmydata/util/mathutil.py | 2 +- src/allmydata/util/netstring.py | 2 +- src/allmydata/util/observer.py | 2 +- src/allmydata/util/pipeline.py | 2 +- src/allmydata/util/pollmixin.py | 2 +- src/allmydata/util/spans.py | 2 +- src/allmydata/util/statistics.py | 2 +- src/allmydata/util/time_format.py | 2 +- src/allmydata/util/verlib.py | 2 +- src/allmydata/version_checks.py | 2 +- 46 files changed, 46 insertions(+), 46 deletions(-) diff --git a/src/allmydata/crypto/__init__.py b/src/allmydata/crypto/__init__.py index 4eb753b57..04b8f0cc3 100644 --- a/src/allmydata/crypto/__init__.py +++ b/src/allmydata/crypto/__init__.py @@ -16,4 +16,4 @@ from __future__ import print_function from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 diff --git a/src/allmydata/crypto/aes.py b/src/allmydata/crypto/aes.py index 196d751bd..ad7cfcba4 100644 --- a/src/allmydata/crypto/aes.py +++ b/src/allmydata/crypto/aes.py @@ -16,7 +16,7 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import six diff --git a/src/allmydata/crypto/ed25519.py b/src/allmydata/crypto/ed25519.py index 424dc23fa..098fa9758 100644 --- a/src/allmydata/crypto/ed25519.py +++ b/src/allmydata/crypto/ed25519.py @@ -24,7 +24,7 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import six diff --git a/src/allmydata/crypto/error.py b/src/allmydata/crypto/error.py index b105805a5..153e48d33 100644 --- a/src/allmydata/crypto/error.py +++ b/src/allmydata/crypto/error.py @@ -10,7 +10,7 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 class BadSignature(Exception): diff --git a/src/allmydata/crypto/rsa.py b/src/allmydata/crypto/rsa.py index 95c39bcc6..b5d15ad4a 100644 --- a/src/allmydata/crypto/rsa.py +++ b/src/allmydata/crypto/rsa.py @@ -19,7 +19,7 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from cryptography.exceptions import InvalidSignature from cryptography.hazmat.backends import default_backend diff --git a/src/allmydata/crypto/util.py b/src/allmydata/crypto/util.py index 6c6ed1059..d377b6396 100644 --- a/src/allmydata/crypto/util.py +++ b/src/allmydata/crypto/util.py @@ -10,7 +10,7 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from allmydata.crypto.error import BadPrefixError diff --git a/src/allmydata/hashtree.py b/src/allmydata/hashtree.py index 77798b3c2..4f8f9d6d7 100644 --- a/src/allmydata/hashtree.py +++ b/src/allmydata/hashtree.py @@ -54,7 +54,7 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from allmydata.util import mathutil # from the pyutil library diff --git a/src/allmydata/test/common_py3.py b/src/allmydata/test/common_py3.py index 97745e293..0dae05aa6 100644 --- a/src/allmydata/test/common_py3.py +++ b/src/allmydata/test/common_py3.py @@ -11,7 +11,7 @@ from __future__ import print_function from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import os import time diff --git a/src/allmydata/test/test_abbreviate.py b/src/allmydata/test/test_abbreviate.py index 958c36742..3ef1e96a6 100644 --- a/src/allmydata/test/test_abbreviate.py +++ b/src/allmydata/test/test_abbreviate.py @@ -10,7 +10,7 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from datetime import timedelta diff --git a/src/allmydata/test/test_base32.py b/src/allmydata/test/test_base32.py index 47636a175..0b9a018b9 100644 --- a/src/allmydata/test/test_base32.py +++ b/src/allmydata/test/test_base32.py @@ -10,7 +10,7 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import base64 diff --git a/src/allmydata/test/test_base62.py b/src/allmydata/test/test_base62.py index e26532e9d..8bbb6dfeb 100644 --- a/src/allmydata/test/test_base62.py +++ b/src/allmydata/test/test_base62.py @@ -11,7 +11,7 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from past.builtins import chr as byteschr diff --git a/src/allmydata/test/test_crypto.py b/src/allmydata/test/test_crypto.py index 498723cf4..0aefa757f 100644 --- a/src/allmydata/test/test_crypto.py +++ b/src/allmydata/test/test_crypto.py @@ -5,7 +5,7 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from future.utils import native_bytes diff --git a/src/allmydata/test/test_deferredutil.py b/src/allmydata/test/test_deferredutil.py index fcff77b54..d8f386e5f 100644 --- a/src/allmydata/test/test_deferredutil.py +++ b/src/allmydata/test/test_deferredutil.py @@ -11,7 +11,7 @@ from __future__ import print_function from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from twisted.trial import unittest from twisted.internet import defer, reactor diff --git a/src/allmydata/test/test_dictutil.py b/src/allmydata/test/test_dictutil.py index 0868db124..9b7124114 100644 --- a/src/allmydata/test/test_dictutil.py +++ b/src/allmydata/test/test_dictutil.py @@ -10,7 +10,7 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from twisted.trial import unittest diff --git a/src/allmydata/test/test_hashtree.py b/src/allmydata/test/test_hashtree.py index b96f4abfb..d1d4cb252 100644 --- a/src/allmydata/test/test_hashtree.py +++ b/src/allmydata/test/test_hashtree.py @@ -11,7 +11,7 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from twisted.trial import unittest diff --git a/src/allmydata/test/test_hashutil.py b/src/allmydata/test/test_hashutil.py index cb0967b83..abcd4f0fb 100644 --- a/src/allmydata/test/test_hashutil.py +++ b/src/allmydata/test/test_hashutil.py @@ -10,7 +10,7 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from twisted.trial import unittest diff --git a/src/allmydata/test/test_humanreadable.py b/src/allmydata/test/test_humanreadable.py index ec6d3f666..94de8f6be 100644 --- a/src/allmydata/test/test_humanreadable.py +++ b/src/allmydata/test/test_humanreadable.py @@ -11,7 +11,7 @@ from __future__ import print_function from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from past.builtins import long diff --git a/src/allmydata/test/test_iputil.py b/src/allmydata/test/test_iputil.py index d3262a9ea..71374fec7 100644 --- a/src/allmydata/test/test_iputil.py +++ b/src/allmydata/test/test_iputil.py @@ -11,7 +11,7 @@ from __future__ import unicode_literals from future.utils import PY2, native_str if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import re, errno, subprocess, os, socket import gc diff --git a/src/allmydata/test/test_netstring.py b/src/allmydata/test/test_netstring.py index b00c99bca..d5ff379cd 100644 --- a/src/allmydata/test/test_netstring.py +++ b/src/allmydata/test/test_netstring.py @@ -10,7 +10,7 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from twisted.trial import unittest diff --git a/src/allmydata/test/test_observer.py b/src/allmydata/test/test_observer.py index b37f0d3e1..0db13db58 100644 --- a/src/allmydata/test/test_observer.py +++ b/src/allmydata/test/test_observer.py @@ -11,7 +11,7 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from twisted.trial import unittest from twisted.internet import defer, reactor diff --git a/src/allmydata/test/test_pipeline.py b/src/allmydata/test/test_pipeline.py index ab7059521..1295be363 100644 --- a/src/allmydata/test/test_pipeline.py +++ b/src/allmydata/test/test_pipeline.py @@ -10,7 +10,7 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import gc diff --git a/src/allmydata/test/test_python3.py b/src/allmydata/test/test_python3.py index 1326c8282..7a6d0b282 100644 --- a/src/allmydata/test/test_python3.py +++ b/src/allmydata/test/test_python3.py @@ -10,7 +10,7 @@ from __future__ import print_function from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from twisted.python.modules import ( getModule, diff --git a/src/allmydata/test/test_spans.py b/src/allmydata/test/test_spans.py index f62d6e684..02d8292f3 100644 --- a/src/allmydata/test/test_spans.py +++ b/src/allmydata/test/test_spans.py @@ -9,7 +9,7 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from past.builtins import long diff --git a/src/allmydata/test/test_statistics.py b/src/allmydata/test/test_statistics.py index 8d5837fc5..476f0a084 100644 --- a/src/allmydata/test/test_statistics.py +++ b/src/allmydata/test/test_statistics.py @@ -10,7 +10,7 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from six.moves import StringIO # native string StringIO diff --git a/src/allmydata/test/test_time_format.py b/src/allmydata/test/test_time_format.py index fa68ccc36..dc9c03b91 100644 --- a/src/allmydata/test/test_time_format.py +++ b/src/allmydata/test/test_time_format.py @@ -8,7 +8,7 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from past.builtins import long diff --git a/src/allmydata/test/test_version.py b/src/allmydata/test/test_version.py index 1b447e346..7301399d9 100644 --- a/src/allmydata/test/test_version.py +++ b/src/allmydata/test/test_version.py @@ -10,7 +10,7 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import sys import pkg_resources diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 87fe265d6..b4b6b0695 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -11,7 +11,7 @@ from __future__ import print_function from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 # Keep these sorted alphabetically, to reduce merge conflicts: PORTED_MODULES = [ diff --git a/src/allmydata/util/abbreviate.py b/src/allmydata/util/abbreviate.py index e7bdd8410..f895c3727 100644 --- a/src/allmydata/util/abbreviate.py +++ b/src/allmydata/util/abbreviate.py @@ -10,7 +10,7 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import re from datetime import timedelta diff --git a/src/allmydata/util/assertutil.py b/src/allmydata/util/assertutil.py index af8817702..ed4b8599f 100644 --- a/src/allmydata/util/assertutil.py +++ b/src/allmydata/util/assertutil.py @@ -14,7 +14,7 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 # The API importers expect: diff --git a/src/allmydata/util/base32.py b/src/allmydata/util/base32.py index ba5f80d67..287d214ea 100644 --- a/src/allmydata/util/base32.py +++ b/src/allmydata/util/base32.py @@ -10,7 +10,7 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 if PY2: def backwardscompat_bytes(b): diff --git a/src/allmydata/util/base62.py b/src/allmydata/util/base62.py index 3d80671ba..964baff34 100644 --- a/src/allmydata/util/base62.py +++ b/src/allmydata/util/base62.py @@ -10,7 +10,7 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 if PY2: import string diff --git a/src/allmydata/util/deferredutil.py b/src/allmydata/util/deferredutil.py index e36892f96..9ce05ef3a 100644 --- a/src/allmydata/util/deferredutil.py +++ b/src/allmydata/util/deferredutil.py @@ -11,7 +11,7 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import time diff --git a/src/allmydata/util/gcutil.py b/src/allmydata/util/gcutil.py index ade95341f..33f1f64f5 100644 --- a/src/allmydata/util/gcutil.py +++ b/src/allmydata/util/gcutil.py @@ -17,7 +17,7 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 __all__ = [ "fileDescriptorResource", diff --git a/src/allmydata/util/hashutil.py b/src/allmydata/util/hashutil.py index d26f5a9b0..96d52c862 100644 --- a/src/allmydata/util/hashutil.py +++ b/src/allmydata/util/hashutil.py @@ -10,7 +10,7 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from past.builtins import chr as byteschr diff --git a/src/allmydata/util/humanreadable.py b/src/allmydata/util/humanreadable.py index 8c7079c3a..60ac57083 100644 --- a/src/allmydata/util/humanreadable.py +++ b/src/allmydata/util/humanreadable.py @@ -11,7 +11,7 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import os from reprlib import Repr diff --git a/src/allmydata/util/iputil.py b/src/allmydata/util/iputil.py index 8754fca53..bd5ea7e78 100644 --- a/src/allmydata/util/iputil.py +++ b/src/allmydata/util/iputil.py @@ -11,7 +11,7 @@ from __future__ import unicode_literals from future.utils import PY2, native_str if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import os, re, socket, subprocess, errno from sys import platform diff --git a/src/allmydata/util/mathutil.py b/src/allmydata/util/mathutil.py index be88f60b4..42863c30e 100644 --- a/src/allmydata/util/mathutil.py +++ b/src/allmydata/util/mathutil.py @@ -13,7 +13,7 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 # The API importers expect: diff --git a/src/allmydata/util/netstring.py b/src/allmydata/util/netstring.py index 7f2bef377..14e515619 100644 --- a/src/allmydata/util/netstring.py +++ b/src/allmydata/util/netstring.py @@ -10,7 +10,7 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from past.builtins import long diff --git a/src/allmydata/util/observer.py b/src/allmydata/util/observer.py index d5003dfb3..4ebb598c1 100644 --- a/src/allmydata/util/observer.py +++ b/src/allmydata/util/observer.py @@ -11,7 +11,7 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import weakref from twisted.internet import defer diff --git a/src/allmydata/util/pipeline.py b/src/allmydata/util/pipeline.py index df80e2c6c..31f5d5d49 100644 --- a/src/allmydata/util/pipeline.py +++ b/src/allmydata/util/pipeline.py @@ -11,7 +11,7 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from twisted.internet import defer from twisted.python.failure import Failure diff --git a/src/allmydata/util/pollmixin.py b/src/allmydata/util/pollmixin.py index 4eb19b8e3..5d1716853 100644 --- a/src/allmydata/util/pollmixin.py +++ b/src/allmydata/util/pollmixin.py @@ -11,7 +11,7 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import time from twisted.internet import task diff --git a/src/allmydata/util/spans.py b/src/allmydata/util/spans.py index 814904d68..b224f0950 100644 --- a/src/allmydata/util/spans.py +++ b/src/allmydata/util/spans.py @@ -5,7 +5,7 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 class Spans(object): diff --git a/src/allmydata/util/statistics.py b/src/allmydata/util/statistics.py index a690eb235..3da5da70a 100644 --- a/src/allmydata/util/statistics.py +++ b/src/allmydata/util/statistics.py @@ -18,7 +18,7 @@ from __future__ import print_function from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from functools import reduce diff --git a/src/allmydata/util/time_format.py b/src/allmydata/util/time_format.py index 5807791e4..ff267485e 100644 --- a/src/allmydata/util/time_format.py +++ b/src/allmydata/util/time_format.py @@ -11,7 +11,7 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from future.utils import native_str import calendar, datetime, re, time diff --git a/src/allmydata/util/verlib.py b/src/allmydata/util/verlib.py index f69e34e3d..2dfc24a1b 100644 --- a/src/allmydata/util/verlib.py +++ b/src/allmydata/util/verlib.py @@ -11,7 +11,7 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import re diff --git a/src/allmydata/version_checks.py b/src/allmydata/version_checks.py index 51a49d78a..d022055ea 100644 --- a/src/allmydata/version_checks.py +++ b/src/allmydata/version_checks.py @@ -11,7 +11,7 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 __all__ = [ "PackagingError", From b8df17283b5956dcd1fdd2a15c0d4468cd4060a6 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 5 Aug 2020 11:54:09 -0400 Subject: [PATCH 38/79] News file. Not bothering with explanation since the regression was only in unreleased versions. --- newsfragments/3368.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3368.minor diff --git a/newsfragments/3368.minor b/newsfragments/3368.minor new file mode 100644 index 000000000..e69de29bb From 9669e4eaa7b121ac6672e6753d127124563ddbb0 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 6 Aug 2020 16:12:26 -0400 Subject: [PATCH 39/79] Drop int. --- src/allmydata/util/dictutil.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/util/dictutil.py b/src/allmydata/util/dictutil.py index d5f0da29b..2a80e7e33 100644 --- a/src/allmydata/util/dictutil.py +++ b/src/allmydata/util/dictutil.py @@ -13,7 +13,7 @@ if PY2: # IMPORTANT: We deliberately don't import dict. The issue is that we're # subclassing dict, so we'd end up exposing Python 3 dict APIs to lots of # code that doesn't support it. - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, list, object, range, str, max, min # noqa: F401 class DictOfSets(dict): From dab690a757f0dab595012438821f3ab61aa5c3d2 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 6 Aug 2020 16:13:10 -0400 Subject: [PATCH 40/79] Flake fixes. --- src/allmydata/test/test_util.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/allmydata/test/test_util.py b/src/allmydata/test/test_util.py index 330abebe5..0a542f444 100644 --- a/src/allmydata/test/test_util.py +++ b/src/allmydata/test/test_util.py @@ -5,7 +5,6 @@ import os, time, sys import yaml from twisted.trial import unittest -from twisted.internet import defer, reactor from twisted.python.failure import Failure from allmydata.util import idlib, mathutil From 32945b85f6d84820843465f39da732ad5170fdde Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 6 Aug 2020 16:14:39 -0400 Subject: [PATCH 41/79] Drop int. --- src/allmydata/test/test_log.py | 2 +- src/allmydata/util/log.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/test_log.py b/src/allmydata/test/test_log.py index 2c26e8ff4..dea198506 100644 --- a/src/allmydata/test/test_log.py +++ b/src/allmydata/test/test_log.py @@ -11,7 +11,7 @@ from __future__ import print_function from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from twisted.trial import unittest from twisted.python.failure import Failure diff --git a/src/allmydata/util/log.py b/src/allmydata/util/log.py index dcb5838a3..11c78a5a2 100644 --- a/src/allmydata/util/log.py +++ b/src/allmydata/util/log.py @@ -10,7 +10,7 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from pyutil import nummedobj From ff5cfe0c71822ff13d39e0567320339dd6205687 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 6 Aug 2020 16:15:31 -0400 Subject: [PATCH 42/79] Update ratchet with new test. --- misc/python3/ratchet-passing | 1 + 1 file changed, 1 insertion(+) diff --git a/misc/python3/ratchet-passing b/misc/python3/ratchet-passing index 2edba65c0..56d82d4e7 100644 --- a/misc/python3/ratchet-passing +++ b/misc/python3/ratchet-passing @@ -60,6 +60,7 @@ allmydata.test.test_log.Log.test_grandparent_id allmydata.test.test_log.Log.test_no_prefix allmydata.test.test_log.Log.test_numming allmydata.test.test_log.Log.test_parent_id +allmydata.test.test_log.Log.test_with_bytes_prefix allmydata.test.test_log.Log.test_with_prefix allmydata.test.test_netstring.Netstring.test_encode allmydata.test.test_netstring.Netstring.test_extra From c6d4ec0295446071c6e1dd8036e876d1ed317137 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 6 Aug 2020 16:33:04 -0400 Subject: [PATCH 43/79] Port the test module to Python 3. --- src/allmydata/test/test_happiness.py | 10 ++++++++++ src/allmydata/util/_python3.py | 1 + 2 files changed, 11 insertions(+) diff --git a/src/allmydata/test/test_happiness.py b/src/allmydata/test/test_happiness.py index a8ed06363..ffba402ef 100644 --- a/src/allmydata/test/test_happiness.py +++ b/src/allmydata/test/test_happiness.py @@ -1,5 +1,15 @@ # -*- coding: utf-8 -*- +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + # We omit dict, just in case newdict breaks things. + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, list, object, range, str, max, min # noqa: F401 + from twisted.trial import unittest from hypothesis import given from hypothesis.strategies import text, sets diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 87fe265d6..be0a382e8 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -52,6 +52,7 @@ PORTED_TEST_MODULES = [ "allmydata.test.test_crypto", "allmydata.test.test_deferredutil", "allmydata.test.test_dictutil", + "allmydata.test.test_happiness", "allmydata.test.test_hashtree", "allmydata.test.test_hashutil", "allmydata.test.test_humanreadable", From d8f74770ebd1a6f27f2a69d24ba22c5a0f83f010 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 6 Aug 2020 16:38:19 -0400 Subject: [PATCH 44/79] Port to Python 3. --- misc/python3/ratchet-passing | 15 +++++++++ newsfragments/3370.minor | 0 src/allmydata/immutable/happiness_upload.py | 35 +++++++++++++-------- src/allmydata/util/_python3.py | 1 + 4 files changed, 38 insertions(+), 13 deletions(-) create mode 100644 newsfragments/3370.minor diff --git a/misc/python3/ratchet-passing b/misc/python3/ratchet-passing index 951603419..48db826f2 100644 --- a/misc/python3/ratchet-passing +++ b/misc/python3/ratchet-passing @@ -57,6 +57,21 @@ allmydata.test.test_deferredutil.DeferredUtilTests.test_success allmydata.test.test_deferredutil.DeferredUtilTests.test_wait_for_delayed_calls allmydata.test.test_dictutil.DictUtil.test_auxdict allmydata.test.test_dictutil.DictUtil.test_dict_of_sets +allmydata.test.test_happiness.Happiness.test_100 +allmydata.test.test_happiness.Happiness.test_calc_happy +allmydata.test.test_happiness.Happiness.test_everything_broken +allmydata.test.test_happiness.Happiness.test_hypothesis0 +allmydata.test.test_happiness.Happiness.test_hypothesis_0 +allmydata.test.test_happiness.Happiness.test_hypothesis_1 +allmydata.test.test_happiness.Happiness.test_placement_1 +allmydata.test.test_happiness.Happiness.test_placement_simple +allmydata.test.test_happiness.Happiness.test_redistribute +allmydata.test.test_happiness.Happiness.test_unhappy +allmydata.test.test_happiness.HappinessUtils.test_residual_0 +allmydata.test.test_happiness.HappinessUtils.test_trivial_flow_graph +allmydata.test.test_happiness.HappinessUtils.test_trivial_maximum_graph +allmydata.test.test_happiness.PlacementTests.test_hypothesis_unhappy +allmydata.test.test_happiness.PlacementTests.test_more_hypothesis allmydata.test.test_hashtree.Complete.test_create allmydata.test.test_hashtree.Complete.test_dump allmydata.test.test_hashtree.Complete.test_needed_hashes diff --git a/newsfragments/3370.minor b/newsfragments/3370.minor new file mode 100644 index 000000000..e69de29bb diff --git a/src/allmydata/immutable/happiness_upload.py b/src/allmydata/immutable/happiness_upload.py index 75edb74d9..a177660cc 100644 --- a/src/allmydata/immutable/happiness_upload.py +++ b/src/allmydata/immutable/happiness_upload.py @@ -1,5 +1,14 @@ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals -from Queue import PriorityQueue +from future.utils import PY2 +if PY2: + # We omit dict, just in case newdict breaks things for external Python 2 code. + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, list, object, range, str, max, min # noqa: F401 + +from queue import PriorityQueue def augmenting_path_for(graph): @@ -35,9 +44,9 @@ def bfs(graph, s): GRAY = 1 # BLACK vertices are those we have seen and explored BLACK = 2 - color = [WHITE for i in xrange(len(graph))] - predecessor = [None for i in xrange(len(graph))] - distance = [-1 for i in xrange(len(graph))] + color = [WHITE for i in range(len(graph))] + predecessor = [None for i in range(len(graph))] + distance = [-1 for i in range(len(graph))] queue = [s] # vertices that we haven't explored yet. color[s] = GRAY distance[s] = 0 @@ -58,9 +67,9 @@ def residual_network(graph, f): flow network represented by my graph and f arguments. graph is a flow network in adjacency-list form, and f is a flow in graph. """ - new_graph = [[] for i in xrange(len(graph))] - cf = [[0 for s in xrange(len(graph))] for sh in xrange(len(graph))] - for i in xrange(len(graph)): + new_graph = [[] for i in range(len(graph))] + cf = [[0 for s in range(len(graph))] for sh in range(len(graph))] + for i in range(len(graph)): for v in graph[i]: if f[i][v] == 1: # We add an edge (v, i) with cf[v,i] = 1. This means @@ -135,7 +144,7 @@ def _compute_maximum_graph(graph, shareIndices): return {} dim = len(graph) - flow_function = [[0 for sh in xrange(dim)] for s in xrange(dim)] + flow_function = [[0 for sh in range(dim)] for s in range(dim)] residual_graph, residual_function = residual_network(graph, flow_function) while augmenting_path_for(residual_graph): @@ -260,9 +269,9 @@ def _servermap_flow_graph(peers, shares, servermap): #print "share_to_index %s" % share_to_index #print "servermap %s" % servermap for peer in peers: - if servermap.has_key(peer): + if peer in servermap: for s in servermap[peer]: - if share_to_index.has_key(s): + if s in share_to_index: indexedShares.append(share_to_index[s]) graph.insert(peer_to_index[peer], indexedShares) for share in shares: @@ -373,7 +382,7 @@ def share_placement(peers, readonly_peers, shares, peers_to_shares): new_mappings = _calculate_mappings(new_peers, new_shares) #print "new_peers %s" % new_peers #print "new_mappings %s" % new_mappings - mappings = dict(readonly_mappings.items() + existing_mappings.items() + new_mappings.items()) + mappings = dict(list(readonly_mappings.items()) + list(existing_mappings.items()) + list(new_mappings.items())) homeless_shares = set() for share in mappings: if mappings[share] is None: @@ -384,7 +393,7 @@ def share_placement(peers, readonly_peers, shares, peers_to_shares): mappings, homeless_shares, { k: v - for k, v in peers_to_shares.items() + for k, v in list(peers_to_shares.items()) if k not in readonly_peers } ) @@ -401,5 +410,5 @@ def share_placement(peers, readonly_peers, shares, peers_to_shares): return { k: v.pop() if v else next(peer_iter) - for k, v in mappings.items() + for k, v in list(mappings.items()) } diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index be0a382e8..d0b2d4a21 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -22,6 +22,7 @@ PORTED_MODULES = [ "allmydata.crypto.rsa", "allmydata.crypto.util", "allmydata.hashtree", + "allmydata.immutable._python3", "allmydata.util.abbreviate", "allmydata.util.assertutil", "allmydata.util.base32", From 49c631b4c8d20bda1bb0f7c6a0d4017e14be3355 Mon Sep 17 00:00:00 2001 From: Chad Whitacre Date: Fri, 7 Aug 2020 08:43:17 -0400 Subject: [PATCH 45/79] Sort the thing we said we'd sort --- newsfragments/3372.minor | 0 src/allmydata/util/_python3.py | 4 ++-- 2 files changed, 2 insertions(+), 2 deletions(-) create mode 100644 newsfragments/3372.minor diff --git a/newsfragments/3372.minor b/newsfragments/3372.minor new file mode 100644 index 000000000..e69de29bb diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index b4b6b0695..ea852cc54 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -22,6 +22,8 @@ PORTED_MODULES = [ "allmydata.crypto.rsa", "allmydata.crypto.util", "allmydata.hashtree", + "allmydata.test.common_py3", + "allmydata.util._python3", "allmydata.util.abbreviate", "allmydata.util.assertutil", "allmydata.util.base32", @@ -38,11 +40,9 @@ PORTED_MODULES = [ "allmydata.util.observer", "allmydata.util.pipeline", "allmydata.util.pollmixin", - "allmydata.util._python3", "allmydata.util.spans", "allmydata.util.statistics", "allmydata.util.time_format", - "allmydata.test.common_py3", ] PORTED_TEST_MODULES = [ From fb9bf5511ff18a407fdfc357fca94ec15d58450f Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 7 Aug 2020 11:09:41 -0400 Subject: [PATCH 46/79] Passing tests. --- src/allmydata/test/test_log.py | 49 +++++++++++++++++++--------------- 1 file changed, 27 insertions(+), 22 deletions(-) diff --git a/src/allmydata/test/test_log.py b/src/allmydata/test/test_log.py index dea198506..eecbda9e3 100644 --- a/src/allmydata/test/test_log.py +++ b/src/allmydata/test/test_log.py @@ -25,22 +25,6 @@ class SampleError(Exception): pass -class LoggingObject0(tahoe_log.PrefixingLogMixin): - pass - - -class LoggingObject1(tahoe_log.PrefixingLogMixin): - pass - - -class LoggingObject2(tahoe_log.PrefixingLogMixin): - pass - - -class LoggingObject3(tahoe_log.PrefixingLogMixin): - pass - - class Log(unittest.TestCase): def setUp(self): self.messages = [] @@ -67,6 +51,9 @@ class Log(unittest.TestCase): If facility is passed to PrefixingLogMixin.__init__, it is used as default facility. """ + class LoggingObject1(tahoe_log.PrefixingLogMixin): + pass + obj = LoggingObject1(facility="defaultfac") obj.log("hello") obj.log("world", facility="override") @@ -78,28 +65,37 @@ class Log(unittest.TestCase): If prefix is passed to PrefixingLogMixin.__init__, it is used in message rendering. """ - obj = LoggingObject0("fac", prefix="pre1") + class LoggingObject4(tahoe_log.PrefixingLogMixin): + pass + + obj = LoggingObject4("fac", prefix="pre1") obj.log("hello") obj.log("world") - self.assertEqual(self.messages[-2][0], '(pre1): hello') - self.assertEqual(self.messages[-1][0], '(pre1): world') + self.assertEqual(self.messages[-2][0], '(pre1): hello') + self.assertEqual(self.messages[-1][0], '(pre1): world') def test_with_bytes_prefix(self): """ If bytes prefix is passed to PrefixingLogMixin.__init__, it is used in message rendering. """ - obj = LoggingObject0("fac", prefix=b"pre1") + class LoggingObject5(tahoe_log.PrefixingLogMixin): + pass + + obj = LoggingObject5("fac", prefix=b"pre1") obj.log("hello") obj.log("world") - self.assertEqual(self.messages[-2][0], '(pre1): hello') - self.assertEqual(self.messages[-1][0], '(pre1): world') + self.assertEqual(self.messages[-2][0], '(pre1): hello') + self.assertEqual(self.messages[-1][0], '(pre1): world') def test_no_prefix(self): """ If no prefix is passed to PrefixingLogMixin.__init__, it is not used in message rendering. """ + class LoggingObject2(tahoe_log.PrefixingLogMixin): + pass + obj = LoggingObject2() obj.log("hello") obj.log("world") @@ -111,6 +107,9 @@ class Log(unittest.TestCase): Objects inheriting from PrefixingLogMixin get a unique number from a class-specific counter. """ + class LoggingObject3(tahoe_log.PrefixingLogMixin): + pass + obj = LoggingObject3() obj2 = LoggingObject3() obj.log("hello") @@ -125,6 +124,9 @@ class Log(unittest.TestCase): This logic is pretty bogus, but that's what the code does. """ + class LoggingObject1(tahoe_log.PrefixingLogMixin): + pass + obj = LoggingObject1() result = obj.log("zero") self.assertEqual(result, "msg1") @@ -140,6 +142,9 @@ class Log(unittest.TestCase): If grandparent message id is given, it's used as parent id of the first message. """ + class LoggingObject1(tahoe_log.PrefixingLogMixin): + pass + obj = LoggingObject1(grandparentmsgid="grand") result = obj.log("zero") self.assertEqual(result, "msg1") From 4d2193fe13bd12ff9d73025bb54dcaf25c7e0bad Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 7 Aug 2020 11:12:12 -0400 Subject: [PATCH 47/79] Fix the module name. --- src/allmydata/util/_python3.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index aa1ab2fb9..608d6ebea 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -22,7 +22,7 @@ PORTED_MODULES = [ "allmydata.crypto.rsa", "allmydata.crypto.util", "allmydata.hashtree", - "allmydata.immutable._python3", + "allmydata.immutable.happiness_upload", "allmydata.util.abbreviate", "allmydata.util.assertutil", "allmydata.util.base32", From 80136da0a60688756be0029402a2fe7b0ff2087d Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 7 Aug 2020 11:14:58 -0400 Subject: [PATCH 48/79] This Fedora probably also has no Python version tag --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 203993f4e..f5b33b106 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -314,7 +314,7 @@ jobs: fedora-29: <<: *RHEL_DERIV docker: - - image: "tahoelafsci/fedora:29-py2" + - image: "tahoelafsci/fedora:29-py" user: "nobody" From d6a80c891a8ec9796760dcf213eb8249f76dfb5d Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 7 Aug 2020 13:39:51 -0400 Subject: [PATCH 49/79] try to understand the CI failures --- misc/python3/ratchet.sh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/misc/python3/ratchet.sh b/misc/python3/ratchet.sh index a6a2c53c3..ec67ea0a0 100755 --- a/misc/python3/ratchet.sh +++ b/misc/python3/ratchet.sh @@ -32,6 +32,11 @@ set -e if [ $TERM = 'dumb' ]; then export TERM=ansi fi -git diff "$tracking_filename" -exit $code +echo "The ${tracking_filename} diff is:" +echo "=================================" +git diff "${tracking_filename}" +echo "=================================" + +echo "Exiting with code ${code} from ratchet.py." +exit ${code} From 5736f9cdebdbd0c4415c354276982c3fb74ab19b Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 7 Aug 2020 14:31:32 -0400 Subject: [PATCH 50/79] Let's just see, for fun --- misc/python3/ratchet.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/python3/ratchet.sh b/misc/python3/ratchet.sh index ec67ea0a0..e38999dd2 100755 --- a/misc/python3/ratchet.sh +++ b/misc/python3/ratchet.sh @@ -35,7 +35,7 @@ fi echo "The ${tracking_filename} diff is:" echo "=================================" -git diff "${tracking_filename}" +git diff -- "${tracking_filename}" echo "=================================" echo "Exiting with code ${code} from ratchet.py." From 1f3d24656750464c43d62e815c7f3a9944b6694d Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 7 Aug 2020 15:13:33 -0400 Subject: [PATCH 51/79] try to get more info --- misc/python3/ratchet.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/misc/python3/ratchet.sh b/misc/python3/ratchet.sh index e38999dd2..da3c88e51 100755 --- a/misc/python3/ratchet.sh +++ b/misc/python3/ratchet.sh @@ -35,6 +35,12 @@ fi echo "The ${tracking_filename} diff is:" echo "=================================" +export GIT_TRACE=1 +export GIT_CURL_VERBOSE=2 +export GIT_TRACE_PACK_ACCESS=1 +export GIT_TRACE_PACKET=1 +export GIT_TRACE_PERFORMANCE=1 +export GIT_TRACE_SETUP=1 git diff -- "${tracking_filename}" echo "=================================" From c2b1fda4e7502c82b1b53fc5cbd8fba22e793d0c Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 7 Aug 2020 14:59:27 -0400 Subject: [PATCH 52/79] Add Ubuntu 20.04 --- .circleci/config.yml | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index d0f596fc7..dae9e904e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -11,10 +11,13 @@ workflows: requires: - "debian-9" - - "ubuntu-18.04" + - "ubuntu-20.04" + - "ubuntu-18.04": + requires: + - "ubuntu-20.04" - "ubuntu-16.04": requires: - - "ubuntu-18.04" + - "ubuntu-20.04" - "fedora-29" - "fedora-28": @@ -65,6 +68,7 @@ workflows: - "build-image-debian-9" - "build-image-ubuntu-16.04" - "build-image-ubuntu-18.04" + - "build-image-ubuntu-20.04" - "build-image-fedora-28" - "build-image-fedora-29" - "build-image-centos-8" @@ -268,6 +272,13 @@ jobs: user: "nobody" + ubuntu-20.04: + <<: *DEBIAN + docker: + - image: "tahoelafsci/ubuntu:20.04" + user: "nobody" + + centos-8: &RHEL_DERIV docker: - image: "tahoelafsci/centos:8" @@ -480,6 +491,14 @@ jobs: TAG: "18.04" + build-image-ubuntu-20.04: + <<: *BUILD_IMAGE + + environment: + DISTRO: "ubuntu" + TAG: "20.04" + + build-image-centos-8: <<: *BUILD_IMAGE From 083be5f0c834e2ca16791d53e49c0beb2916af6c Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 7 Aug 2020 14:59:45 -0400 Subject: [PATCH 53/79] news fragment --- newsfragments/3328.installation | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/3328.installation diff --git a/newsfragments/3328.installation b/newsfragments/3328.installation new file mode 100644 index 000000000..7b08ffdc4 --- /dev/null +++ b/newsfragments/3328.installation @@ -0,0 +1 @@ +Tahoe-LAFS now supports Ubuntu 20.04. \ No newline at end of file From b1e67774b427fe57678a403192aed2fe450ef27d Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 7 Aug 2020 15:46:46 -0400 Subject: [PATCH 54/79] Try getting the _trial_temp out of the working tree --- misc/python3/ratchet.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/python3/ratchet.sh b/misc/python3/ratchet.sh index da3c88e51..65a270560 100755 --- a/misc/python3/ratchet.sh +++ b/misc/python3/ratchet.sh @@ -11,7 +11,7 @@ cd "../.." # Since both of the next calls are expected to exit non-0, relax our guard. set +e -SUBUNITREPORTER_OUTPUT_PATH="$base/results.subunit2" trial --reporter subunitv2-file allmydata +SUBUNITREPORTER_OUTPUT_PATH="$base/results.subunit2" trial --temp-directory /tmp/_trial_temp.ratchet --reporter subunitv2-file allmydata subunit2junitxml < "$base/results.subunit2" > "$base/results.xml" set -e From 7c922785245e93fd61398289e9328e3ae5cf47ac Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 7 Aug 2020 15:52:57 -0400 Subject: [PATCH 55/79] getting ridiculous --- misc/python3/ratchet.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/python3/ratchet.sh b/misc/python3/ratchet.sh index 65a270560..c92b22c5d 100755 --- a/misc/python3/ratchet.sh +++ b/misc/python3/ratchet.sh @@ -41,7 +41,7 @@ export GIT_TRACE_PACK_ACCESS=1 export GIT_TRACE_PACKET=1 export GIT_TRACE_PERFORMANCE=1 export GIT_TRACE_SETUP=1 -git diff -- "${tracking_filename}" +strace git diff -- "${tracking_filename}" echo "=================================" echo "Exiting with code ${code} from ratchet.py." From 8b44b89f7534efe02e386c2b883b33ed442da5a5 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 7 Aug 2020 18:33:35 -0400 Subject: [PATCH 56/79] oi --- misc/python3/ratchet.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/misc/python3/ratchet.sh b/misc/python3/ratchet.sh index c92b22c5d..9d57a0e10 100755 --- a/misc/python3/ratchet.sh +++ b/misc/python3/ratchet.sh @@ -33,6 +33,7 @@ if [ $TERM = 'dumb' ]; then export TERM=ansi fi +apt-get install -y strace echo "The ${tracking_filename} diff is:" echo "=================================" export GIT_TRACE=1 From 107861802bf46681a211195256a2bec3f24e4335 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 7 Aug 2020 21:18:29 -0400 Subject: [PATCH 57/79] idk maybe this --- misc/python3/ratchet.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/python3/ratchet.sh b/misc/python3/ratchet.sh index 9d57a0e10..3d824d199 100755 --- a/misc/python3/ratchet.sh +++ b/misc/python3/ratchet.sh @@ -33,7 +33,7 @@ if [ $TERM = 'dumb' ]; then export TERM=ansi fi -apt-get install -y strace +sudo apt-get install -y strace echo "The ${tracking_filename} diff is:" echo "=================================" export GIT_TRACE=1 From a60d4f7df4c799e09c261015b5091f534e306f28 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 7 Aug 2020 21:56:37 -0400 Subject: [PATCH 58/79] get strace --- .circleci/Dockerfile.ubuntu | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/Dockerfile.ubuntu b/.circleci/Dockerfile.ubuntu index 2fcc60f5a..98c3b5088 100644 --- a/.circleci/Dockerfile.ubuntu +++ b/.circleci/Dockerfile.ubuntu @@ -13,6 +13,7 @@ RUN apt-get --quiet update && \ apt-get --quiet --yes install git && \ apt-get --quiet --yes install \ sudo \ + strace \ build-essential \ python${PYTHON_VERSION} \ python${PYTHON_VERSION}-dev \ From a758b84e15126cf28ac0969a0f4c36db57d0713b Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 7 Aug 2020 21:56:51 -0400 Subject: [PATCH 59/79] handling this elsewhere --- misc/python3/ratchet.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/misc/python3/ratchet.sh b/misc/python3/ratchet.sh index 3d824d199..c92b22c5d 100755 --- a/misc/python3/ratchet.sh +++ b/misc/python3/ratchet.sh @@ -33,7 +33,6 @@ if [ $TERM = 'dumb' ]; then export TERM=ansi fi -sudo apt-get install -y strace echo "The ${tracking_filename} diff is:" echo "=================================" export GIT_TRACE=1 From 9af1e741d8566a6f4bd8817531fafe1b510d022c Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 7 Aug 2020 22:19:38 -0400 Subject: [PATCH 60/79] that was kind of annoyingly non-illuminating --- misc/python3/ratchet.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/python3/ratchet.sh b/misc/python3/ratchet.sh index c92b22c5d..6c428cdf9 100755 --- a/misc/python3/ratchet.sh +++ b/misc/python3/ratchet.sh @@ -41,7 +41,7 @@ export GIT_TRACE_PACK_ACCESS=1 export GIT_TRACE_PACKET=1 export GIT_TRACE_PERFORMANCE=1 export GIT_TRACE_SETUP=1 -strace git diff -- "${tracking_filename}" +strace -f git diff -- "${tracking_filename}" echo "=================================" echo "Exiting with code ${code} from ratchet.py." From 8d1798fbb6c8186ca346525ba0bf1fec98d2493a Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 7 Aug 2020 22:26:39 -0400 Subject: [PATCH 61/79] is the pager screwing up? --- misc/python3/ratchet.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/misc/python3/ratchet.sh b/misc/python3/ratchet.sh index 6c428cdf9..1d54a9cb0 100755 --- a/misc/python3/ratchet.sh +++ b/misc/python3/ratchet.sh @@ -41,7 +41,8 @@ export GIT_TRACE_PACK_ACCESS=1 export GIT_TRACE_PACKET=1 export GIT_TRACE_PERFORMANCE=1 export GIT_TRACE_SETUP=1 -strace -f git diff -- "${tracking_filename}" +strace -f git diff -- "${tracking_filename}" > tracking.diff +cat tracking.diff echo "=================================" echo "Exiting with code ${code} from ratchet.py." From fe850e724000970bff75a74797f99b33863f213e Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 7 Aug 2020 22:45:33 -0400 Subject: [PATCH 62/79] redirecting to a file seems to help --- misc/python3/ratchet.sh | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/misc/python3/ratchet.sh b/misc/python3/ratchet.sh index 1d54a9cb0..c05e675f2 100755 --- a/misc/python3/ratchet.sh +++ b/misc/python3/ratchet.sh @@ -35,13 +35,9 @@ fi echo "The ${tracking_filename} diff is:" echo "=================================" -export GIT_TRACE=1 -export GIT_CURL_VERBOSE=2 -export GIT_TRACE_PACK_ACCESS=1 -export GIT_TRACE_PACKET=1 -export GIT_TRACE_PERFORMANCE=1 -export GIT_TRACE_SETUP=1 -strace -f git diff -- "${tracking_filename}" > tracking.diff +# "git diff" gets pretty confused in this execution context when trying to +# write to stdout. Somehow it fails with SIGTTOU. +git diff -- "${tracking_filename}" > tracking.diff cat tracking.diff echo "=================================" From ef3cbebe431b9978d211434dc6c141d3c26699eb Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 7 Aug 2020 22:57:59 -0400 Subject: [PATCH 63/79] Perhaps get this path right --- misc/python3/ratchet.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/misc/python3/ratchet.sh b/misc/python3/ratchet.sh index c05e675f2..33183fce1 100755 --- a/misc/python3/ratchet.sh +++ b/misc/python3/ratchet.sh @@ -11,8 +11,8 @@ cd "../.." # Since both of the next calls are expected to exit non-0, relax our guard. set +e -SUBUNITREPORTER_OUTPUT_PATH="$base/results.subunit2" trial --temp-directory /tmp/_trial_temp.ratchet --reporter subunitv2-file allmydata -subunit2junitxml < "$base/results.subunit2" > "$base/results.xml" +trial --temp-directory /tmp/_trial_temp.ratchet --reporter subunitv2-file allmydata +subunit2junitxml < "${SUBUNITREPORTER_OUTPUT_PATH}" > "$base/results.xml" set -e # Okay, now we're clear. From 360a97a5794bd69a210d72f8a411412d47413a83 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 7 Aug 2020 23:08:48 -0400 Subject: [PATCH 64/79] Let the python36 job have the env vars --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index ab467abd0..27c9f36cc 100644 --- a/tox.ini +++ b/tox.ini @@ -50,7 +50,7 @@ commands = [testenv:py36] # git inside of ratchet.sh needs $HOME. -passenv = HOME +passenv = {[testenv]passenv} HOME commands = {toxinidir}/misc/python3/ratchet.sh [testenv:integration] From 65520786cac9415af6ccfb71fa3e858a751b806e Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 7 Aug 2020 23:26:15 -0400 Subject: [PATCH 65/79] [drop] faster ci --- .circleci/config.yml | 64 ++++++++++++++++++++++---------------------- 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index f5b33b106..1ae4c1326 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -6,50 +6,50 @@ workflows: ci: jobs: # Platforms - - "debian-9" - - "debian-8": - requires: - - "debian-9" + # - "debian-9" + # - "debian-8": + # requires: + # - "debian-9" - - "ubuntu-18.04" - - "ubuntu-16.04": - requires: - - "ubuntu-18.04" + # - "ubuntu-18.04" + # - "ubuntu-16.04": + # requires: + # - "ubuntu-18.04" - - "fedora-29" - - "fedora-28": - requires: - - "fedora-29" + # - "fedora-29" + # - "fedora-28": + # requires: + # - "fedora-29" - - "centos-8" + # - "centos-8" - - "slackware-14.2" + # - "slackware-14.2" - - "nixos-19.09" + # - "nixos-19.09" - # Test against PyPy 2.7 - - "pypy2.7-buster" + # # Test against PyPy 2.7 + # - "pypy2.7-buster" # Just one Python 3.6 configuration while the port is in-progress. - "python3.6" - # Other assorted tasks and configurations - - "lint" - - "pyinstaller" - - "deprecations" - - "c-locale" - # Any locale other than C or UTF-8. - - "another-locale" + # # Other assorted tasks and configurations + # - "lint" + # - "pyinstaller" + # - "deprecations" + # - "c-locale" + # # Any locale other than C or UTF-8. + # - "another-locale" - - "integration": - requires: - # If the unit test suite doesn't pass, don't bother running the - # integration tests. - - "debian-9" + # - "integration": + # requires: + # # If the unit test suite doesn't pass, don't bother running the + # # integration tests. + # - "debian-9" - # Generate the underlying data for a visualization to aid with Python 3 - # porting. - - "build-porting-depgraph" + # # Generate the underlying data for a visualization to aid with Python 3 + # # porting. + # - "build-porting-depgraph" images: # Build the Docker images used by the ci jobs. This makes the ci jobs From f8e44acdef3bb0aa06bc8d74a881bf7b69b6a77c Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 7 Aug 2020 23:27:36 -0400 Subject: [PATCH 66/79] did it get written --- misc/python3/ratchet.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/misc/python3/ratchet.sh b/misc/python3/ratchet.sh index 33183fce1..c9bf6bed1 100755 --- a/misc/python3/ratchet.sh +++ b/misc/python3/ratchet.sh @@ -12,6 +12,7 @@ cd "../.." # Since both of the next calls are expected to exit non-0, relax our guard. set +e trial --temp-directory /tmp/_trial_temp.ratchet --reporter subunitv2-file allmydata +find "${SUBUNITREPORTER_OUTPUT_PATH}" subunit2junitxml < "${SUBUNITREPORTER_OUTPUT_PATH}" > "$base/results.xml" set -e From c17337e732797d36bb46253b43f17b008ad86b07 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 7 Aug 2020 23:47:11 -0400 Subject: [PATCH 67/79] We get passed an env var that tells us how to run trial Also temp-dir made no difference and don't need the find either. --- misc/python3/ratchet.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/misc/python3/ratchet.sh b/misc/python3/ratchet.sh index c9bf6bed1..e9f8a4c04 100755 --- a/misc/python3/ratchet.sh +++ b/misc/python3/ratchet.sh @@ -11,8 +11,7 @@ cd "../.." # Since both of the next calls are expected to exit non-0, relax our guard. set +e -trial --temp-directory /tmp/_trial_temp.ratchet --reporter subunitv2-file allmydata -find "${SUBUNITREPORTER_OUTPUT_PATH}" +trial ${TAHOE_LAFS_TRIAL_ARGS} allmydata subunit2junitxml < "${SUBUNITREPORTER_OUTPUT_PATH}" > "$base/results.xml" set -e From 971bca7402706f9b864f5d5e4746212775a629dd Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 7 Aug 2020 23:51:15 -0400 Subject: [PATCH 68/79] --rterrors is broken with subunitreporter on python 3 --- misc/python3/ratchet.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/python3/ratchet.sh b/misc/python3/ratchet.sh index e9f8a4c04..aa768cd06 100755 --- a/misc/python3/ratchet.sh +++ b/misc/python3/ratchet.sh @@ -11,7 +11,7 @@ cd "../.." # Since both of the next calls are expected to exit non-0, relax our guard. set +e -trial ${TAHOE_LAFS_TRIAL_ARGS} allmydata +trial --reporter=subunitv2-file allmydata subunit2junitxml < "${SUBUNITREPORTER_OUTPUT_PATH}" > "$base/results.xml" set -e From 23d7ffa369060286e6bcbe516288681dc980ed2c Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 7 Aug 2020 23:56:13 -0400 Subject: [PATCH 69/79] Check for subunit2 output here --- .circleci/run-tests.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.circleci/run-tests.sh b/.circleci/run-tests.sh index c26acdcbc..aab3271d6 100755 --- a/.circleci/run-tests.sh +++ b/.circleci/run-tests.sh @@ -81,6 +81,11 @@ ${TIMEOUT} ${BOOTSTRAP_VENV}/bin/tox \ ${TAHOE_LAFS_TOX_ARGS} || "${alternative}" if [ -n "${ARTIFACTS}" ]; then + if [ ! -e "${SUBUNIT2}" ]; then + echo "subunitv2 output file does not exist: ${SUBUNIT2}" + exit 1 + fi + # Create a junitxml results area. mkdir -p "$(dirname "${JUNITXML}")" ${BOOTSTRAP_VENV}/bin/subunit2junitxml < "${SUBUNIT2}" > "${JUNITXML}" || "${alternative}" From 7ba6daeed52aed906076c33ecb7946ee900de980 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 7 Aug 2020 23:58:24 -0400 Subject: [PATCH 70/79] maybe get some more visibility into errors --- .circleci/run-tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/run-tests.sh b/.circleci/run-tests.sh index aab3271d6..939f9b1e5 100755 --- a/.circleci/run-tests.sh +++ b/.circleci/run-tests.sh @@ -88,5 +88,5 @@ if [ -n "${ARTIFACTS}" ]; then # Create a junitxml results area. mkdir -p "$(dirname "${JUNITXML}")" - ${BOOTSTRAP_VENV}/bin/subunit2junitxml < "${SUBUNIT2}" > "${JUNITXML}" || "${alternative}" + cat "${SUBUNIT2}" | ${BOOTSTRAP_VENV}/bin/subunit2junitxml | cat > "${JUNITXML}" || "${alternative}" fi From a927ebbafaaf80fbb22b4cdd5bc0e57dca0958ce Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Sat, 8 Aug 2020 00:04:13 -0400 Subject: [PATCH 71/79] what's up --- .circleci/run-tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/run-tests.sh b/.circleci/run-tests.sh index 939f9b1e5..197227442 100755 --- a/.circleci/run-tests.sh +++ b/.circleci/run-tests.sh @@ -88,5 +88,5 @@ if [ -n "${ARTIFACTS}" ]; then # Create a junitxml results area. mkdir -p "$(dirname "${JUNITXML}")" - cat "${SUBUNIT2}" | ${BOOTSTRAP_VENV}/bin/subunit2junitxml | cat > "${JUNITXML}" || "${alternative}" + cat "${SUBUNIT2}" | ${BOOTSTRAP_VENV}/bin/subunit2junitxml fi From 0c2cf9f0c5fbc6b501af782f87bd741ad1f6d068 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 10 Aug 2020 09:11:14 -0400 Subject: [PATCH 72/79] Don't fail if subunit2junitxml fails --- .circleci/run-tests.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.circleci/run-tests.sh b/.circleci/run-tests.sh index 197227442..8894b9691 100755 --- a/.circleci/run-tests.sh +++ b/.circleci/run-tests.sh @@ -88,5 +88,9 @@ if [ -n "${ARTIFACTS}" ]; then # Create a junitxml results area. mkdir -p "$(dirname "${JUNITXML}")" - cat "${SUBUNIT2}" | ${BOOTSTRAP_VENV}/bin/subunit2junitxml + # Always succeed even if subunit2junitxml fails. subunit2junitxml signals + # failure if the stream it is processing contains test failures. This is + # not what we care about. If we cared about it, the test command above + # would have signalled failure already and we wouldn't be here. + "${BOOTSTRAP_VENV}"/bin/subunit2junitxml < "${SUBUNIT2}" > "${JUNITXML}" || true fi From 462e212a19f93c833ab308f7d8f5bb2ed848e035 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 10 Aug 2020 09:16:19 -0400 Subject: [PATCH 73/79] Revert "[drop] faster ci" This reverts commit 65520786cac9415af6ccfb71fa3e858a751b806e. --- .circleci/config.yml | 64 ++++++++++++++++++++++---------------------- 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1ae4c1326..f5b33b106 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -6,50 +6,50 @@ workflows: ci: jobs: # Platforms - # - "debian-9" - # - "debian-8": - # requires: - # - "debian-9" + - "debian-9" + - "debian-8": + requires: + - "debian-9" - # - "ubuntu-18.04" - # - "ubuntu-16.04": - # requires: - # - "ubuntu-18.04" + - "ubuntu-18.04" + - "ubuntu-16.04": + requires: + - "ubuntu-18.04" - # - "fedora-29" - # - "fedora-28": - # requires: - # - "fedora-29" + - "fedora-29" + - "fedora-28": + requires: + - "fedora-29" - # - "centos-8" + - "centos-8" - # - "slackware-14.2" + - "slackware-14.2" - # - "nixos-19.09" + - "nixos-19.09" - # # Test against PyPy 2.7 - # - "pypy2.7-buster" + # Test against PyPy 2.7 + - "pypy2.7-buster" # Just one Python 3.6 configuration while the port is in-progress. - "python3.6" - # # Other assorted tasks and configurations - # - "lint" - # - "pyinstaller" - # - "deprecations" - # - "c-locale" - # # Any locale other than C or UTF-8. - # - "another-locale" + # Other assorted tasks and configurations + - "lint" + - "pyinstaller" + - "deprecations" + - "c-locale" + # Any locale other than C or UTF-8. + - "another-locale" - # - "integration": - # requires: - # # If the unit test suite doesn't pass, don't bother running the - # # integration tests. - # - "debian-9" + - "integration": + requires: + # If the unit test suite doesn't pass, don't bother running the + # integration tests. + - "debian-9" - # # Generate the underlying data for a visualization to aid with Python 3 - # # porting. - # - "build-porting-depgraph" + # Generate the underlying data for a visualization to aid with Python 3 + # porting. + - "build-porting-depgraph" images: # Build the Docker images used by the ci jobs. This makes the ci jobs From fdfa95e1e8b5bbc2aeadd9b999cc5fb57515beea Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 10 Aug 2020 09:59:28 -0400 Subject: [PATCH 74/79] news fragment --- newsfragments/3375.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3375.minor diff --git a/newsfragments/3375.minor b/newsfragments/3375.minor new file mode 100644 index 000000000..e69de29bb From 9853fa90904be82e71c7e9fd19d99d2177289bad Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 10 Aug 2020 09:59:36 -0400 Subject: [PATCH 75/79] Pin Python 2 compatible PyInstaller --- tox.ini | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 4356cfd24..908d24d54 100644 --- a/tox.ini +++ b/tox.ini @@ -210,7 +210,9 @@ extras = deps = {[testenv]deps} packaging - pyinstaller + # PyInstaller 4.0 drops Python 2 support. When we finish porting to + # Python 3 we can reconsider this constraint. + pyinstaller < 4.0 # Setting PYTHONHASHSEED to a known value assists with reproducible builds. # See https://pyinstaller.readthedocs.io/en/stable/advanced-topics.html#creating-a-reproducible-build setenv=PYTHONHASHSEED=1 From 0f65d10f47abf761d27bd4fb1e12e4d442553472 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 11 Aug 2020 13:25:08 -0400 Subject: [PATCH 76/79] news fragment --- newsfragments/3323.removed | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/3323.removed diff --git a/newsfragments/3323.removed b/newsfragments/3323.removed new file mode 100644 index 000000000..356b4b2af --- /dev/null +++ b/newsfragments/3323.removed @@ -0,0 +1 @@ +Slackware 14.2 is no longer a Tahoe-LAFS supported platform. From a74ef86098c8a2e9676c226d6861903c2fac7a9d Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 11 Aug 2020 13:25:16 -0400 Subject: [PATCH 77/79] Remove the CI configuration --- .circleci/Dockerfile.slackware | 49 ---------------------------------- .circleci/config.yml | 31 --------------------- 2 files changed, 80 deletions(-) delete mode 100644 .circleci/Dockerfile.slackware diff --git a/.circleci/Dockerfile.slackware b/.circleci/Dockerfile.slackware deleted file mode 100644 index 73ba6b32d..000000000 --- a/.circleci/Dockerfile.slackware +++ /dev/null @@ -1,49 +0,0 @@ -ARG TAG -FROM vbatts/slackware:${TAG} - -ENV WHEELHOUSE_PATH /tmp/wheelhouse -ENV VIRTUALENV_PATH /tmp/venv -# This will get updated by the CircleCI checkout step. -ENV BUILD_SRC_ROOT /tmp/project - -# Be careful with slackpkg. If the package name given doesn't match anything, -# slackpkg still claims to succeed but you're totally screwed. Slackware -# updates versions of packaged software so including too much version prefix -# is a good way to have your install commands suddenly begin not installing -# anything. -RUN slackpkg update && \ - slackpkg install \ - openssh-7 git-2 \ - ca-certificates \ - sudo-1 \ - make-4 \ - automake-1 \ - kernel-headers \ - glibc-2 \ - binutils-2 \ - gcc-5 \ - gcc-g++-5 \ - python-2 \ - libffi-3 \ - libyaml-0 \ - sqlite-3 \ - icu4c-56 \ - libmpc-1 Date: Tue, 11 Aug 2020 14:54:12 -0400 Subject: [PATCH 78/79] Docstring. --- src/allmydata/immutable/happiness_upload.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/allmydata/immutable/happiness_upload.py b/src/allmydata/immutable/happiness_upload.py index a177660cc..9716aaef2 100644 --- a/src/allmydata/immutable/happiness_upload.py +++ b/src/allmydata/immutable/happiness_upload.py @@ -1,3 +1,9 @@ +""" +Algorithms for figuring out happiness, the number of unique nodes the data is +on. + +Ported to Python 3. +""" from __future__ import absolute_import from __future__ import division from __future__ import print_function From 513177c4b86469874aa93d3fdbe33a34f6a9ecc2 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 11 Aug 2020 18:33:01 -0400 Subject: [PATCH 79/79] strace not used by anything now --- .circleci/Dockerfile.ubuntu | 1 - 1 file changed, 1 deletion(-) diff --git a/.circleci/Dockerfile.ubuntu b/.circleci/Dockerfile.ubuntu index 98c3b5088..2fcc60f5a 100644 --- a/.circleci/Dockerfile.ubuntu +++ b/.circleci/Dockerfile.ubuntu @@ -13,7 +13,6 @@ RUN apt-get --quiet update && \ apt-get --quiet --yes install git && \ apt-get --quiet --yes install \ sudo \ - strace \ build-essential \ python${PYTHON_VERSION} \ python${PYTHON_VERSION}-dev \