diff --git a/.circleci/Dockerfile.centos b/.circleci/Dockerfile.centos index febb61545..9070d71d9 100644 --- a/.circleci/Dockerfile.centos +++ b/.circleci/Dockerfile.centos @@ -1,5 +1,6 @@ ARG TAG FROM centos:${TAG} +ARG PYTHON_VERSION ENV WHEELHOUSE_PATH /tmp/wheelhouse ENV VIRTUALENV_PATH /tmp/venv @@ -11,8 +12,8 @@ RUN yum install --assumeyes \ git \ sudo \ make automake gcc gcc-c++ \ - python2 \ - python2-devel \ + python${PYTHON_VERSION} \ + python${PYTHON_VERSION}-devel \ libffi-devel \ openssl-devel \ libyaml \ @@ -23,4 +24,4 @@ RUN yum install --assumeyes \ # *update* this checkout on each job run, saving us more time per-job. COPY . ${BUILD_SRC_ROOT} -RUN "${BUILD_SRC_ROOT}"/.circleci/prepare-image.sh "${WHEELHOUSE_PATH}" "${VIRTUALENV_PATH}" "${BUILD_SRC_ROOT}" "python2.7" +RUN "${BUILD_SRC_ROOT}"/.circleci/prepare-image.sh "${WHEELHOUSE_PATH}" "${VIRTUALENV_PATH}" "${BUILD_SRC_ROOT}" "python${PYTHON_VERSION}" diff --git a/.circleci/Dockerfile.debian b/.circleci/Dockerfile.debian index de16148e0..96c54736c 100644 --- a/.circleci/Dockerfile.debian +++ b/.circleci/Dockerfile.debian @@ -1,5 +1,6 @@ ARG TAG FROM debian:${TAG} +ARG PYTHON_VERSION ENV WHEELHOUSE_PATH /tmp/wheelhouse ENV VIRTUALENV_PATH /tmp/venv @@ -8,22 +9,22 @@ ENV BUILD_SRC_ROOT /tmp/project RUN apt-get --quiet update && \ apt-get --quiet --yes install \ - git \ - lsb-release \ + git \ + lsb-release \ sudo \ - build-essential \ - python2.7 \ - python2.7-dev \ - libffi-dev \ - libssl-dev \ - libyaml-dev \ - virtualenv + build-essential \ + python${PYTHON_VERSION} \ + python${PYTHON_VERSION}-dev \ + libffi-dev \ + libssl-dev \ + libyaml-dev \ + virtualenv # Get the project source. This is better than it seems. CircleCI will # *update* this checkout on each job run, saving us more time per-job. COPY . ${BUILD_SRC_ROOT} -RUN "${BUILD_SRC_ROOT}"/.circleci/prepare-image.sh "${WHEELHOUSE_PATH}" "${VIRTUALENV_PATH}" "${BUILD_SRC_ROOT}" "python2.7" +RUN "${BUILD_SRC_ROOT}"/.circleci/prepare-image.sh "${WHEELHOUSE_PATH}" "${VIRTUALENV_PATH}" "${BUILD_SRC_ROOT}" "python${PYTHON_VERSION}" # Only the integration tests currently need this but it doesn't hurt to always # have it present and it's simpler than building a whole extra image just for diff --git a/.circleci/Dockerfile.fedora b/.circleci/Dockerfile.fedora index 6ad22d676..e60dbb85d 100644 --- a/.circleci/Dockerfile.fedora +++ b/.circleci/Dockerfile.fedora @@ -1,5 +1,6 @@ ARG TAG FROM fedora:${TAG} +ARG PYTHON_VERSION ENV WHEELHOUSE_PATH /tmp/wheelhouse ENV VIRTUALENV_PATH /tmp/venv @@ -11,8 +12,8 @@ RUN yum install --assumeyes \ git \ sudo \ make automake gcc gcc-c++ \ - python \ - python-devel \ + python${PYTHON_VERSION} \ + python${PYTHON_VERSION}-devel \ libffi-devel \ openssl-devel \ libyaml-devel \ @@ -23,4 +24,4 @@ RUN yum install --assumeyes \ # *update* this checkout on each job run, saving us more time per-job. COPY . ${BUILD_SRC_ROOT} -RUN "${BUILD_SRC_ROOT}"/.circleci/prepare-image.sh "${WHEELHOUSE_PATH}" "${VIRTUALENV_PATH}" "${BUILD_SRC_ROOT}" "python2.7" +RUN "${BUILD_SRC_ROOT}"/.circleci/prepare-image.sh "${WHEELHOUSE_PATH}" "${VIRTUALENV_PATH}" "${BUILD_SRC_ROOT}" "python${PYTHON_VERSION}" diff --git a/.circleci/Dockerfile.slackware b/.circleci/Dockerfile.slackware deleted file mode 100644 index 73ba6b32d..000000000 --- a/.circleci/Dockerfile.slackware +++ /dev/null @@ -1,49 +0,0 @@ -ARG TAG -FROM vbatts/slackware:${TAG} - -ENV WHEELHOUSE_PATH /tmp/wheelhouse -ENV VIRTUALENV_PATH /tmp/venv -# This will get updated by the CircleCI checkout step. -ENV BUILD_SRC_ROOT /tmp/project - -# Be careful with slackpkg. If the package name given doesn't match anything, -# slackpkg still claims to succeed but you're totally screwed. Slackware -# updates versions of packaged software so including too much version prefix -# is a good way to have your install commands suddenly begin not installing -# anything. -RUN slackpkg update && \ - slackpkg install \ - openssh-7 git-2 \ - ca-certificates \ - sudo-1 \ - make-4 \ - automake-1 \ - kernel-headers \ - glibc-2 \ - binutils-2 \ - gcc-5 \ - gcc-g++-5 \ - python-2 \ - libffi-3 \ - libyaml-0 \ - sqlite-3 \ - icu4c-56 \ - libmpc-1 :foo" - TAG: "tahoelafsci/distro:" + DISTRO: "tahoelafsci/:foo-py2" + TAG: "tahoelafsci/distro:-py2" + PYTHON_VERSION: "tahoelafsci/distro:tag-py "${JUNITXML}" || "${alternative}" + # Always succeed even if subunit2junitxml fails. subunit2junitxml signals + # failure if the stream it is processing contains test failures. This is + # not what we care about. If we cared about it, the test command above + # would have signalled failure already and we wouldn't be here. + "${BOOTSTRAP_VENV}"/bin/subunit2junitxml < "${SUBUNIT2}" > "${JUNITXML}" || true fi diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 40f351ae6..000000000 --- a/.travis.yml +++ /dev/null @@ -1,40 +0,0 @@ -sudo: false -language: python -cache: pip -dist: xenial -before_cache: - - rm -f $HOME/.cache/pip/log/debug.log -git: - depth: 1000 - -env: - global: - - TAHOE_LAFS_HYPOTHESIS_PROFILE=ci - -install: - - pip install --upgrade tox setuptools virtualenv - - echo $PATH; which python; which pip; which tox - - python misc/build_helpers/show-tool-versions.py - -script: - - | - set -eo pipefail - tox -e ${T} - -notifications: - email: false - irc: - channels: "chat.freenode.net#tahoe-lafs" - on_success: always # for testing - on_failure: always - template: - - "%{repository}#%{build_number} [%{branch}: %{commit} by %{author}] %{message}" - - "Changes: %{compare_url} | Details: %{build_url}" - -matrix: - include: - - os: linux - python: '3.6' - env: T=py36 - - fast_finish: true diff --git a/misc/python3/ratchet-passing b/misc/python3/ratchet-passing index 62cd5d0cc..200126859 100644 --- a/misc/python3/ratchet-passing +++ b/misc/python3/ratchet-passing @@ -27,12 +27,54 @@ allmydata.test.test_base62.Base62.test_known_values allmydata.test.test_base62.Base62.test_num_octets_that_encode_to_this_many_chars allmydata.test.test_base62.Base62.test_odd_sizes allmydata.test.test_base62.Base62.test_roundtrip +allmydata.test.test_crypto.TestEd25519.test_deserialize_private_not_bytes +allmydata.test.test_crypto.TestEd25519.test_deserialize_public_not_bytes +allmydata.test.test_crypto.TestEd25519.test_key_serialization +allmydata.test.test_crypto.TestEd25519.test_sign_invalid_pubkey +allmydata.test.test_crypto.TestEd25519.test_signature_data_not_bytes +allmydata.test.test_crypto.TestEd25519.test_signature_not_bytes +allmydata.test.test_crypto.TestEd25519.test_signed_data_not_bytes +allmydata.test.test_crypto.TestEd25519.test_verify_invalid_pubkey +allmydata.test.test_crypto.TestRegression.test_aes_no_iv_process_long_input +allmydata.test.test_crypto.TestRegression.test_aes_no_iv_process_short_input +allmydata.test.test_crypto.TestRegression.test_aes_with_iv_process_long_input +allmydata.test.test_crypto.TestRegression.test_aes_with_iv_process_short_input +allmydata.test.test_crypto.TestRegression.test_decode_ed15519_keypair +allmydata.test.test_crypto.TestRegression.test_decode_rsa_keypair +allmydata.test.test_crypto.TestRegression.test_encrypt_data_not_bytes +allmydata.test.test_crypto.TestRegression.test_incorrect_iv_size +allmydata.test.test_crypto.TestRegression.test_iv_not_bytes +allmydata.test.test_crypto.TestRegression.test_key_incorrect_size +allmydata.test.test_crypto.TestRegression.test_old_start_up_test +allmydata.test.test_crypto.TestRsa.test_keys +allmydata.test.test_crypto.TestRsa.test_sign_invalid_pubkey +allmydata.test.test_crypto.TestRsa.test_verify_invalid_pubkey +allmydata.test.test_crypto.TestUtil.test_remove_prefix_bad +allmydata.test.test_crypto.TestUtil.test_remove_prefix_entire_string +allmydata.test.test_crypto.TestUtil.test_remove_prefix_good +allmydata.test.test_crypto.TestUtil.test_remove_prefix_partial +allmydata.test.test_crypto.TestUtil.test_remove_prefix_zero allmydata.test.test_deferredutil.DeferredUtilTests.test_failure allmydata.test.test_deferredutil.DeferredUtilTests.test_gather_results allmydata.test.test_deferredutil.DeferredUtilTests.test_success allmydata.test.test_deferredutil.DeferredUtilTests.test_wait_for_delayed_calls allmydata.test.test_dictutil.DictUtil.test_auxdict allmydata.test.test_dictutil.DictUtil.test_dict_of_sets +allmydata.test.test_happiness.Happiness.test_100 +allmydata.test.test_happiness.Happiness.test_calc_happy +allmydata.test.test_happiness.Happiness.test_everything_broken +allmydata.test.test_happiness.Happiness.test_hypothesis0 +allmydata.test.test_happiness.Happiness.test_hypothesis_0 +allmydata.test.test_happiness.Happiness.test_hypothesis_1 +allmydata.test.test_happiness.Happiness.test_placement_1 +allmydata.test.test_happiness.Happiness.test_placement_simple +allmydata.test.test_happiness.Happiness.test_redistribute +allmydata.test.test_happiness.Happiness.test_unhappy +allmydata.test.test_happiness.HappinessUtils.test_residual_0 +allmydata.test.test_happiness.HappinessUtils.test_trivial_flow_graph +allmydata.test.test_happiness.HappinessUtils.test_trivial_maximum_graph +allmydata.test.test_happiness.PlacementTests.test_hypothesis_unhappy +allmydata.test.test_happiness.PlacementTests.test_more_hypothesis allmydata.test.test_hashtree.Complete.test_create allmydata.test.test_hashtree.Complete.test_dump allmydata.test.test_hashtree.Complete.test_needed_hashes @@ -49,6 +91,8 @@ allmydata.test.test_hashutil.HashUtilTests.test_sha256d allmydata.test.test_hashutil.HashUtilTests.test_sha256d_truncated allmydata.test.test_hashutil.HashUtilTests.test_timing_safe_compare allmydata.test.test_humanreadable.HumanReadable.test_repr +allmydata.test.test_iputil.GcUtil.test_gc_after_allocations +allmydata.test.test_iputil.GcUtil.test_release_delays_gc allmydata.test.test_iputil.ListAddresses.test_get_local_ip_for allmydata.test.test_iputil.ListAddresses.test_list_async allmydata.test.test_iputil.ListAddresses.test_list_async_mock_cygwin @@ -57,6 +101,14 @@ allmydata.test.test_iputil.ListAddresses.test_list_async_mock_ip_addr allmydata.test.test_iputil.ListAddresses.test_list_async_mock_route allmydata.test.test_iputil.ListenOnUsed.test_random_port allmydata.test.test_iputil.ListenOnUsed.test_specific_port +allmydata.test.test_log.Log.test_default_facility +allmydata.test.test_log.Log.test_err +allmydata.test.test_log.Log.test_grandparent_id +allmydata.test.test_log.Log.test_no_prefix +allmydata.test.test_log.Log.test_numming +allmydata.test.test_log.Log.test_parent_id +allmydata.test.test_log.Log.test_with_bytes_prefix +allmydata.test.test_log.Log.test_with_prefix allmydata.test.test_netstring.Netstring.test_encode allmydata.test.test_netstring.Netstring.test_extra allmydata.test.test_netstring.Netstring.test_nested diff --git a/misc/python3/ratchet.sh b/misc/python3/ratchet.sh index aeffe2ff2..aa768cd06 100755 --- a/misc/python3/ratchet.sh +++ b/misc/python3/ratchet.sh @@ -11,9 +11,8 @@ cd "../.." # Since both of the next calls are expected to exit non-0, relax our guard. set +e -SUBUNITREPORTER_OUTPUT_PATH="$base/results.subunit2" trial --reporter subunitv2-file allmydata -subunit2junitxml < "$base/results.subunit2" > "$base/results.xml" -subunit2pyunit < "$base/results.subunit2" +trial --reporter=subunitv2-file allmydata +subunit2junitxml < "${SUBUNITREPORTER_OUTPUT_PATH}" > "$base/results.xml" set -e # Okay, now we're clear. @@ -33,6 +32,14 @@ set -e if [ $TERM = 'dumb' ]; then export TERM=ansi fi -git diff "$tracking_filename" -exit $code +echo "The ${tracking_filename} diff is:" +echo "=================================" +# "git diff" gets pretty confused in this execution context when trying to +# write to stdout. Somehow it fails with SIGTTOU. +git diff -- "${tracking_filename}" > tracking.diff +cat tracking.diff +echo "=================================" + +echo "Exiting with code ${code} from ratchet.py." +exit ${code} diff --git a/newsfragments/3323.removed b/newsfragments/3323.removed new file mode 100644 index 000000000..356b4b2af --- /dev/null +++ b/newsfragments/3323.removed @@ -0,0 +1 @@ +Slackware 14.2 is no longer a Tahoe-LAFS supported platform. diff --git a/newsfragments/3326.minor b/newsfragments/3326.minor new file mode 100644 index 000000000..e69de29bb diff --git a/newsfragments/3328.installation b/newsfragments/3328.installation new file mode 100644 index 000000000..7b08ffdc4 --- /dev/null +++ b/newsfragments/3328.installation @@ -0,0 +1 @@ +Tahoe-LAFS now supports Ubuntu 20.04. \ No newline at end of file diff --git a/newsfragments/3336.minor b/newsfragments/3336.minor new file mode 100644 index 000000000..e69de29bb diff --git a/newsfragments/3361.minor b/newsfragments/3361.minor new file mode 100644 index 000000000..e69de29bb diff --git a/newsfragments/3364.minor b/newsfragments/3364.minor new file mode 100644 index 000000000..e69de29bb diff --git a/newsfragments/3365.minor b/newsfragments/3365.minor new file mode 100644 index 000000000..e69de29bb diff --git a/newsfragments/3366.minor b/newsfragments/3366.minor new file mode 100644 index 000000000..e69de29bb diff --git a/newsfragments/3368.minor b/newsfragments/3368.minor new file mode 100644 index 000000000..e69de29bb diff --git a/newsfragments/3370.minor b/newsfragments/3370.minor new file mode 100644 index 000000000..e69de29bb diff --git a/newsfragments/3372.minor b/newsfragments/3372.minor new file mode 100644 index 000000000..e69de29bb diff --git a/newsfragments/3375.minor b/newsfragments/3375.minor new file mode 100644 index 000000000..e69de29bb diff --git a/setup.cfg b/setup.cfg index c2d224225..f4539279e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -7,6 +7,8 @@ bdist_egg = update_version bdist_egg bdist_wheel = update_version bdist_wheel [flake8] -# For now, only use pyflakes errors; flake8 is still helpful because it allows -# ignoring specific errors/warnings when needed. -select = F \ No newline at end of file +# Enforce all pyflakes constraints, and also prohibit tabs for indentation. +# Reference: +# https://flake8.pycqa.org/en/latest/user/error-codes.html +# https://pycodestyle.pycqa.org/en/latest/intro.html#error-codes +select = F, W191 diff --git a/src/allmydata/crypto/__init__.py b/src/allmydata/crypto/__init__.py index ee92f223a..04b8f0cc3 100644 --- a/src/allmydata/crypto/__init__.py +++ b/src/allmydata/crypto/__init__.py @@ -5,4 +5,15 @@ For the most part, these functions use and return objects that are documented in the `cryptography` library -- however, code inside Tahoe should only use these functions and not rely on features of any objects that `cryptography` documents. + +Ported to Python 3. """ + +from __future__ import unicode_literals +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from future.utils import PY2 +if PY2: + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 diff --git a/src/allmydata/crypto/aes.py b/src/allmydata/crypto/aes.py index 4194c63df..ad7cfcba4 100644 --- a/src/allmydata/crypto/aes.py +++ b/src/allmydata/crypto/aes.py @@ -6,7 +6,17 @@ These functions use and return objects that are documented in the `cryptography` library -- however, code inside Tahoe should only use functions from allmydata.crypto.aes and not rely on features of any objects that `cryptography` documents. + +Ported to Python 3. """ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import six diff --git a/src/allmydata/crypto/ed25519.py b/src/allmydata/crypto/ed25519.py index 37e305d19..098fa9758 100644 --- a/src/allmydata/crypto/ed25519.py +++ b/src/allmydata/crypto/ed25519.py @@ -13,7 +13,18 @@ cut-and-pasteability. The base62 encoding is shorter than the base32 form, but the minor usability improvement is not worth the documentation and specification confusion of using a non-standard encoding. So we stick with base32. + +Ported to Python 3. ''' +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + + +from future.utils import PY2 +if PY2: + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import six diff --git a/src/allmydata/crypto/error.py b/src/allmydata/crypto/error.py index 62c0b3e5b..153e48d33 100644 --- a/src/allmydata/crypto/error.py +++ b/src/allmydata/crypto/error.py @@ -1,6 +1,16 @@ """ Exceptions raise by allmydata.crypto.* modules + +Ported to Python 3. """ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 class BadSignature(Exception): diff --git a/src/allmydata/crypto/rsa.py b/src/allmydata/crypto/rsa.py index e82bf12d1..b5d15ad4a 100644 --- a/src/allmydata/crypto/rsa.py +++ b/src/allmydata/crypto/rsa.py @@ -9,8 +9,17 @@ features of any objects that `cryptography` documents. That is, the public and private keys are opaque objects; DO NOT depend on any of their methods. -""" +Ported to Python 3. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from cryptography.exceptions import InvalidSignature from cryptography.hazmat.backends import default_backend diff --git a/src/allmydata/crypto/util.py b/src/allmydata/crypto/util.py index 6aa1f0973..d377b6396 100644 --- a/src/allmydata/crypto/util.py +++ b/src/allmydata/crypto/util.py @@ -1,6 +1,16 @@ """ Utilities used by allmydata.crypto modules + +Ported to Python 3. """ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from allmydata.crypto.error import BadPrefixError diff --git a/src/allmydata/hashtree.py b/src/allmydata/hashtree.py index 77798b3c2..226a7b12b 100644 --- a/src/allmydata/hashtree.py +++ b/src/allmydata/hashtree.py @@ -45,6 +45,8 @@ Written by Connelly Barnes in 2005 and released into the public domain with no warranty of any kind, either expressed or implied. It probably won't make your computer catch on fire, or eat your children, but it might. Use at your own risk. + +Ported to Python 3. """ from __future__ import absolute_import @@ -54,7 +56,7 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from allmydata.util import mathutil # from the pyutil library diff --git a/src/allmydata/immutable/happiness_upload.py b/src/allmydata/immutable/happiness_upload.py index 75edb74d9..9716aaef2 100644 --- a/src/allmydata/immutable/happiness_upload.py +++ b/src/allmydata/immutable/happiness_upload.py @@ -1,5 +1,20 @@ +""" +Algorithms for figuring out happiness, the number of unique nodes the data is +on. -from Queue import PriorityQueue +Ported to Python 3. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + # We omit dict, just in case newdict breaks things for external Python 2 code. + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, list, object, range, str, max, min # noqa: F401 + +from queue import PriorityQueue def augmenting_path_for(graph): @@ -35,9 +50,9 @@ def bfs(graph, s): GRAY = 1 # BLACK vertices are those we have seen and explored BLACK = 2 - color = [WHITE for i in xrange(len(graph))] - predecessor = [None for i in xrange(len(graph))] - distance = [-1 for i in xrange(len(graph))] + color = [WHITE for i in range(len(graph))] + predecessor = [None for i in range(len(graph))] + distance = [-1 for i in range(len(graph))] queue = [s] # vertices that we haven't explored yet. color[s] = GRAY distance[s] = 0 @@ -58,9 +73,9 @@ def residual_network(graph, f): flow network represented by my graph and f arguments. graph is a flow network in adjacency-list form, and f is a flow in graph. """ - new_graph = [[] for i in xrange(len(graph))] - cf = [[0 for s in xrange(len(graph))] for sh in xrange(len(graph))] - for i in xrange(len(graph)): + new_graph = [[] for i in range(len(graph))] + cf = [[0 for s in range(len(graph))] for sh in range(len(graph))] + for i in range(len(graph)): for v in graph[i]: if f[i][v] == 1: # We add an edge (v, i) with cf[v,i] = 1. This means @@ -135,7 +150,7 @@ def _compute_maximum_graph(graph, shareIndices): return {} dim = len(graph) - flow_function = [[0 for sh in xrange(dim)] for s in xrange(dim)] + flow_function = [[0 for sh in range(dim)] for s in range(dim)] residual_graph, residual_function = residual_network(graph, flow_function) while augmenting_path_for(residual_graph): @@ -260,9 +275,9 @@ def _servermap_flow_graph(peers, shares, servermap): #print "share_to_index %s" % share_to_index #print "servermap %s" % servermap for peer in peers: - if servermap.has_key(peer): + if peer in servermap: for s in servermap[peer]: - if share_to_index.has_key(s): + if s in share_to_index: indexedShares.append(share_to_index[s]) graph.insert(peer_to_index[peer], indexedShares) for share in shares: @@ -373,7 +388,7 @@ def share_placement(peers, readonly_peers, shares, peers_to_shares): new_mappings = _calculate_mappings(new_peers, new_shares) #print "new_peers %s" % new_peers #print "new_mappings %s" % new_mappings - mappings = dict(readonly_mappings.items() + existing_mappings.items() + new_mappings.items()) + mappings = dict(list(readonly_mappings.items()) + list(existing_mappings.items()) + list(new_mappings.items())) homeless_shares = set() for share in mappings: if mappings[share] is None: @@ -384,7 +399,7 @@ def share_placement(peers, readonly_peers, shares, peers_to_shares): mappings, homeless_shares, { k: v - for k, v in peers_to_shares.items() + for k, v in list(peers_to_shares.items()) if k not in readonly_peers } ) @@ -401,5 +416,5 @@ def share_placement(peers, readonly_peers, shares, peers_to_shares): return { k: v.pop() if v else next(peer_iter) - for k, v in mappings.items() + for k, v in list(mappings.items()) } diff --git a/src/allmydata/test/common_py3.py b/src/allmydata/test/common_py3.py index 97745e293..0dae05aa6 100644 --- a/src/allmydata/test/common_py3.py +++ b/src/allmydata/test/common_py3.py @@ -11,7 +11,7 @@ from __future__ import print_function from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import os import time diff --git a/src/allmydata/test/test_abbreviate.py b/src/allmydata/test/test_abbreviate.py index 958c36742..3ef1e96a6 100644 --- a/src/allmydata/test/test_abbreviate.py +++ b/src/allmydata/test/test_abbreviate.py @@ -10,7 +10,7 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from datetime import timedelta diff --git a/src/allmydata/test/test_base32.py b/src/allmydata/test/test_base32.py index 47636a175..0b9a018b9 100644 --- a/src/allmydata/test/test_base32.py +++ b/src/allmydata/test/test_base32.py @@ -10,7 +10,7 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import base64 diff --git a/src/allmydata/test/test_base62.py b/src/allmydata/test/test_base62.py index e26532e9d..8bbb6dfeb 100644 --- a/src/allmydata/test/test_base62.py +++ b/src/allmydata/test/test_base62.py @@ -11,7 +11,7 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from past.builtins import chr as byteschr diff --git a/src/allmydata/test/test_crypto.py b/src/allmydata/test/test_crypto.py index 53ba344a4..0aefa757f 100644 --- a/src/allmydata/test/test_crypto.py +++ b/src/allmydata/test/test_crypto.py @@ -1,4 +1,14 @@ -import six +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + +from future.utils import native_bytes + import unittest from base64 import b64decode @@ -37,17 +47,18 @@ class TestRegression(unittest.TestCase): # priv = rsa.generate(2048) # priv_str = b64encode(priv.serialize()) # pub_str = b64encode(priv.get_verifying_key().serialize()) - RSA_2048_PRIV_KEY = six.b(b64decode(f.read().strip())) + RSA_2048_PRIV_KEY = b64decode(f.read().strip()) + assert isinstance(RSA_2048_PRIV_KEY, native_bytes) with RESOURCE_DIR.child('pycryptopp-rsa-2048-sig.txt').open('r') as f: # Signature created using `RSA_2048_PRIV_KEY` via: # # sig = priv.sign(b'test') - RSA_2048_SIG = six.b(b64decode(f.read().strip())) + RSA_2048_SIG = b64decode(f.read().strip()) with RESOURCE_DIR.child('pycryptopp-rsa-2048-pub.txt').open('r') as f: # The public key corresponding to `RSA_2048_PRIV_KEY`. - RSA_2048_PUB_KEY = six.b(b64decode(f.read().strip())) + RSA_2048_PUB_KEY = b64decode(f.read().strip()) def test_old_start_up_test(self): """ @@ -283,7 +294,7 @@ class TestEd25519(unittest.TestCase): private_key, public_key = ed25519.create_signing_keypair() private_key_str = ed25519.string_from_signing_key(private_key) - self.assertIsInstance(private_key_str, six.string_types) + self.assertIsInstance(private_key_str, native_bytes) private_key2, public_key2 = ed25519.signing_keypair_from_string(private_key_str) @@ -299,7 +310,7 @@ class TestEd25519(unittest.TestCase): # ditto, but for the verifying keys public_key_str = ed25519.string_from_verifying_key(public_key) - self.assertIsInstance(public_key_str, six.string_types) + self.assertIsInstance(public_key_str, native_bytes) public_key2 = ed25519.verifying_key_from_string(public_key_str) self.assertEqual( @@ -403,7 +414,7 @@ class TestRsa(unittest.TestCase): priv_key, pub_key = rsa.create_signing_keypair(2048) priv_key_str = rsa.der_string_from_signing_key(priv_key) - self.assertIsInstance(priv_key_str, six.string_types) + self.assertIsInstance(priv_key_str, native_bytes) priv_key2, pub_key2 = rsa.create_signing_keypair_from_string(priv_key_str) diff --git a/src/allmydata/test/test_deferredutil.py b/src/allmydata/test/test_deferredutil.py index fcff77b54..d8f386e5f 100644 --- a/src/allmydata/test/test_deferredutil.py +++ b/src/allmydata/test/test_deferredutil.py @@ -11,7 +11,7 @@ from __future__ import print_function from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from twisted.trial import unittest from twisted.internet import defer, reactor diff --git a/src/allmydata/test/test_dictutil.py b/src/allmydata/test/test_dictutil.py index 0868db124..9b7124114 100644 --- a/src/allmydata/test/test_dictutil.py +++ b/src/allmydata/test/test_dictutil.py @@ -10,7 +10,7 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from twisted.trial import unittest diff --git a/src/allmydata/test/test_happiness.py b/src/allmydata/test/test_happiness.py index a8ed06363..ffba402ef 100644 --- a/src/allmydata/test/test_happiness.py +++ b/src/allmydata/test/test_happiness.py @@ -1,5 +1,15 @@ # -*- coding: utf-8 -*- +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + # We omit dict, just in case newdict breaks things. + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, list, object, range, str, max, min # noqa: F401 + from twisted.trial import unittest from hypothesis import given from hypothesis.strategies import text, sets diff --git a/src/allmydata/test/test_hashtree.py b/src/allmydata/test/test_hashtree.py index b96f4abfb..d1d4cb252 100644 --- a/src/allmydata/test/test_hashtree.py +++ b/src/allmydata/test/test_hashtree.py @@ -11,7 +11,7 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from twisted.trial import unittest diff --git a/src/allmydata/test/test_hashutil.py b/src/allmydata/test/test_hashutil.py index cb0967b83..abcd4f0fb 100644 --- a/src/allmydata/test/test_hashutil.py +++ b/src/allmydata/test/test_hashutil.py @@ -10,7 +10,7 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from twisted.trial import unittest diff --git a/src/allmydata/test/test_humanreadable.py b/src/allmydata/test/test_humanreadable.py index ec6d3f666..94de8f6be 100644 --- a/src/allmydata/test/test_humanreadable.py +++ b/src/allmydata/test/test_humanreadable.py @@ -11,7 +11,7 @@ from __future__ import print_function from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from past.builtins import long diff --git a/src/allmydata/test/test_iputil.py b/src/allmydata/test/test_iputil.py index c6caed7f9..71374fec7 100644 --- a/src/allmydata/test/test_iputil.py +++ b/src/allmydata/test/test_iputil.py @@ -11,9 +11,10 @@ from __future__ import unicode_literals from future.utils import PY2, native_str if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import re, errno, subprocess, os, socket +import gc from twisted.trial import unittest @@ -21,7 +22,7 @@ from tenacity import retry, stop_after_attempt from foolscap.api import Tub -from allmydata.util import iputil +from allmydata.util import iputil, gcutil import allmydata.test.common_py3 as testutil from allmydata.util.namespace import Namespace @@ -228,3 +229,35 @@ class ListenOnUsed(unittest.TestCase): s.close() port2 = iputil.listenOnUnused(tub, port) self.assertEqual(port, port2) + + +class GcUtil(unittest.TestCase): + """Tests for allmydata.util.gcutil, which is used only by listenOnUnused.""" + + def test_gc_after_allocations(self): + """The resource tracker triggers allocations every 26 allocations.""" + tracker = gcutil._ResourceTracker() + collections = [] + self.patch(gc, "collect", lambda: collections.append(1)) + for _ in range(2): + for _ in range(25): + tracker.allocate() + self.assertEqual(len(collections), 0) + tracker.allocate() + self.assertEqual(len(collections), 1) + del collections[:] + + def test_release_delays_gc(self): + """Releasing a file descriptor resource delays GC collection.""" + tracker = gcutil._ResourceTracker() + collections = [] + self.patch(gc, "collect", lambda: collections.append(1)) + for _ in range(2): + tracker.allocate() + for _ in range(3): + tracker.release() + for _ in range(25): + tracker.allocate() + self.assertEqual(len(collections), 0) + tracker.allocate() + self.assertEqual(len(collections), 1) diff --git a/src/allmydata/test/test_log.py b/src/allmydata/test/test_log.py new file mode 100644 index 000000000..eecbda9e3 --- /dev/null +++ b/src/allmydata/test/test_log.py @@ -0,0 +1,156 @@ +""" +Tests for allmydata.util.log. + +Ported to Python 3. +""" + +from __future__ import unicode_literals +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from future.utils import PY2 +if PY2: + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + +from twisted.trial import unittest +from twisted.python.failure import Failure + +from foolscap.logging import log + +from allmydata.util import log as tahoe_log + + +class SampleError(Exception): + pass + + +class Log(unittest.TestCase): + def setUp(self): + self.messages = [] + + def msg(msg, facility, parent, *args, **kwargs): + self.messages.append((msg, facility, parent, args, kwargs)) + return "msg{}".format(len(self.messages)) + + self.patch(log, "msg", msg) + + def test_err(self): + """Logging with log.err() causes tests to fail.""" + try: + raise SampleError("simple sample") + except: + f = Failure() + tahoe_log.err(format="intentional sample error", + failure=f, level=tahoe_log.OPERATIONAL, umid="wO9UoQ") + result = self.flushLoggedErrors(SampleError) + self.assertEqual(len(result), 1) + + def test_default_facility(self): + """ + If facility is passed to PrefixingLogMixin.__init__, it is used as + default facility. + """ + class LoggingObject1(tahoe_log.PrefixingLogMixin): + pass + + obj = LoggingObject1(facility="defaultfac") + obj.log("hello") + obj.log("world", facility="override") + self.assertEqual(self.messages[-2][1], "defaultfac") + self.assertEqual(self.messages[-1][1], "override") + + def test_with_prefix(self): + """ + If prefix is passed to PrefixingLogMixin.__init__, it is used in + message rendering. + """ + class LoggingObject4(tahoe_log.PrefixingLogMixin): + pass + + obj = LoggingObject4("fac", prefix="pre1") + obj.log("hello") + obj.log("world") + self.assertEqual(self.messages[-2][0], '(pre1): hello') + self.assertEqual(self.messages[-1][0], '(pre1): world') + + def test_with_bytes_prefix(self): + """ + If bytes prefix is passed to PrefixingLogMixin.__init__, it is used in + message rendering. + """ + class LoggingObject5(tahoe_log.PrefixingLogMixin): + pass + + obj = LoggingObject5("fac", prefix=b"pre1") + obj.log("hello") + obj.log("world") + self.assertEqual(self.messages[-2][0], '(pre1): hello') + self.assertEqual(self.messages[-1][0], '(pre1): world') + + def test_no_prefix(self): + """ + If no prefix is passed to PrefixingLogMixin.__init__, it is not used in + message rendering. + """ + class LoggingObject2(tahoe_log.PrefixingLogMixin): + pass + + obj = LoggingObject2() + obj.log("hello") + obj.log("world") + self.assertEqual(self.messages[-2][0], ': hello') + self.assertEqual(self.messages[-1][0], ': world') + + def test_numming(self): + """ + Objects inheriting from PrefixingLogMixin get a unique number from a + class-specific counter. + """ + class LoggingObject3(tahoe_log.PrefixingLogMixin): + pass + + obj = LoggingObject3() + obj2 = LoggingObject3() + obj.log("hello") + obj2.log("world") + self.assertEqual(self.messages[-2][0], ': hello') + self.assertEqual(self.messages[-1][0], ': world') + + def test_parent_id(self): + """ + The parent message id can be passed in, otherwise the first message's + id is used as the parent. + + This logic is pretty bogus, but that's what the code does. + """ + class LoggingObject1(tahoe_log.PrefixingLogMixin): + pass + + obj = LoggingObject1() + result = obj.log("zero") + self.assertEqual(result, "msg1") + obj.log("one", parent="par1") + obj.log("two", parent="par2") + obj.log("three") + obj.log("four") + self.assertEqual([m[2] for m in self.messages], + [None, "par1", "par2", "msg1", "msg1"]) + + def test_grandparent_id(self): + """ + If grandparent message id is given, it's used as parent id of the first + message. + """ + class LoggingObject1(tahoe_log.PrefixingLogMixin): + pass + + obj = LoggingObject1(grandparentmsgid="grand") + result = obj.log("zero") + self.assertEqual(result, "msg1") + obj.log("one", parent="par1") + obj.log("two", parent="par2") + obj.log("three") + obj.log("four") + self.assertEqual([m[2] for m in self.messages], + ["grand", "par1", "par2", "msg1", "msg1"]) diff --git a/src/allmydata/test/test_netstring.py b/src/allmydata/test/test_netstring.py index b00c99bca..d5ff379cd 100644 --- a/src/allmydata/test/test_netstring.py +++ b/src/allmydata/test/test_netstring.py @@ -10,7 +10,7 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from twisted.trial import unittest diff --git a/src/allmydata/test/test_observer.py b/src/allmydata/test/test_observer.py index b37f0d3e1..0db13db58 100644 --- a/src/allmydata/test/test_observer.py +++ b/src/allmydata/test/test_observer.py @@ -11,7 +11,7 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from twisted.trial import unittest from twisted.internet import defer, reactor diff --git a/src/allmydata/test/test_pipeline.py b/src/allmydata/test/test_pipeline.py index ab7059521..1295be363 100644 --- a/src/allmydata/test/test_pipeline.py +++ b/src/allmydata/test/test_pipeline.py @@ -10,7 +10,7 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import gc diff --git a/src/allmydata/test/test_python3.py b/src/allmydata/test/test_python3.py index 1326c8282..7a6d0b282 100644 --- a/src/allmydata/test/test_python3.py +++ b/src/allmydata/test/test_python3.py @@ -10,7 +10,7 @@ from __future__ import print_function from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from twisted.python.modules import ( getModule, diff --git a/src/allmydata/test/test_spans.py b/src/allmydata/test/test_spans.py index f62d6e684..02d8292f3 100644 --- a/src/allmydata/test/test_spans.py +++ b/src/allmydata/test/test_spans.py @@ -9,7 +9,7 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from past.builtins import long diff --git a/src/allmydata/test/test_statistics.py b/src/allmydata/test/test_statistics.py index 8d5837fc5..476f0a084 100644 --- a/src/allmydata/test/test_statistics.py +++ b/src/allmydata/test/test_statistics.py @@ -10,7 +10,7 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from six.moves import StringIO # native string StringIO diff --git a/src/allmydata/test/test_time_format.py b/src/allmydata/test/test_time_format.py index fa68ccc36..dc9c03b91 100644 --- a/src/allmydata/test/test_time_format.py +++ b/src/allmydata/test/test_time_format.py @@ -8,7 +8,7 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from past.builtins import long diff --git a/src/allmydata/test/test_util.py b/src/allmydata/test/test_util.py index aa1c716a6..f1f2b1c66 100644 --- a/src/allmydata/test/test_util.py +++ b/src/allmydata/test/test_util.py @@ -15,14 +15,11 @@ import os, time, sys import yaml from twisted.trial import unittest -from twisted.internet import defer, reactor -from twisted.python.failure import Failure from allmydata.util import idlib, mathutil from allmydata.util import fileutil -from allmydata.util import limiter, pollmixin +from allmydata.util import pollmixin from allmydata.util import yamlutil -from allmydata.util import log as tahoe_log from allmydata.util.fileutil import EncryptedTemporaryFile from allmydata.test.common_util import ReallyEqualMixin @@ -441,81 +438,6 @@ class PollMixinTests(unittest.TestCase): return d -class Limiter(unittest.TestCase): - - def job(self, i, foo): - self.calls.append( (i, foo) ) - self.simultaneous += 1 - self.peak_simultaneous = max(self.simultaneous, self.peak_simultaneous) - d = defer.Deferred() - def _done(): - self.simultaneous -= 1 - d.callback("done %d" % i) - reactor.callLater(1.0, _done) - return d - - def bad_job(self, i, foo): - raise ValueError("bad_job %d" % i) - - def test_limiter(self): - self.calls = [] - self.simultaneous = 0 - self.peak_simultaneous = 0 - l = limiter.ConcurrencyLimiter() - dl = [] - for i in range(20): - dl.append(l.add(self.job, i, foo=str(i))) - d = defer.DeferredList(dl, fireOnOneErrback=True) - def _done(res): - self.failUnlessEqual(self.simultaneous, 0) - self.failUnless(self.peak_simultaneous <= 10) - self.failUnlessEqual(len(self.calls), 20) - for i in range(20): - self.failUnless( (i, str(i)) in self.calls) - d.addCallback(_done) - return d - - def test_errors(self): - self.calls = [] - self.simultaneous = 0 - self.peak_simultaneous = 0 - l = limiter.ConcurrencyLimiter() - dl = [] - for i in range(20): - dl.append(l.add(self.job, i, foo=str(i))) - d2 = l.add(self.bad_job, 21, "21") - d = defer.DeferredList(dl, fireOnOneErrback=True) - def _most_done(res): - results = [] - for (success, result) in res: - self.failUnlessEqual(success, True) - results.append(result) - results.sort() - expected_results = ["done %d" % i for i in range(20)] - expected_results.sort() - self.failUnlessEqual(results, expected_results) - self.failUnless(self.peak_simultaneous <= 10) - self.failUnlessEqual(len(self.calls), 20) - for i in range(20): - self.failUnless( (i, str(i)) in self.calls) - def _good(res): - self.fail("should have failed, not got %s" % (res,)) - def _err(f): - f.trap(ValueError) - self.failUnless("bad_job 21" in str(f)) - d2.addCallbacks(_good, _err) - return d2 - d.addCallback(_most_done) - def _all_done(res): - self.failUnlessEqual(self.simultaneous, 0) - self.failUnless(self.peak_simultaneous <= 10) - self.failUnlessEqual(len(self.calls), 20) - for i in range(20): - self.failUnless( (i, str(i)) in self.calls) - d.addCallback(_all_done) - return d - - ctr = [0] class EqButNotIs(object): def __init__(self, x): @@ -540,20 +462,6 @@ class EqButNotIs(object): return self.x == other -class SampleError(Exception): - pass - -class Log(unittest.TestCase): - def test_err(self): - try: - raise SampleError("simple sample") - except: - f = Failure() - tahoe_log.err(format="intentional sample error", - failure=f, level=tahoe_log.OPERATIONAL, umid="wO9UoQ") - self.flushLoggedErrors(SampleError) - - class YAML(unittest.TestCase): def test_convert(self): data = yaml.safe_dump(["str", u"unicode", u"\u1234nicode"]) diff --git a/src/allmydata/test/test_version.py b/src/allmydata/test/test_version.py index 1b447e346..7301399d9 100644 --- a/src/allmydata/test/test_version.py +++ b/src/allmydata/test/test_version.py @@ -10,7 +10,7 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import sys import pkg_resources diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 62e252406..a1750ca18 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -11,11 +11,20 @@ from __future__ import print_function from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 # Keep these sorted alphabetically, to reduce merge conflicts: PORTED_MODULES = [ + "allmydata.crypto", + "allmydata.crypto.aes", + "allmydata.crypto.ed25519", + "allmydata.crypto.error", + "allmydata.crypto.rsa", + "allmydata.crypto.util", "allmydata.hashtree", + "allmydata.immutable.happiness_upload", + "allmydata.test.common_py3", + "allmydata.util._python3", "allmydata.util.abbreviate", "allmydata.util.assertutil", "allmydata.util.base32", @@ -23,32 +32,35 @@ PORTED_MODULES = [ "allmydata.util.deferredutil", "allmydata.util.fileutil", "allmydata.util.dictutil", + "allmydata.util.gcutil", "allmydata.util.hashutil", "allmydata.util.humanreadable", "allmydata.util.iputil", + "allmydata.util.log", "allmydata.util.mathutil", "allmydata.util.namespace", "allmydata.util.netstring", "allmydata.util.observer", "allmydata.util.pipeline", "allmydata.util.pollmixin", - "allmydata.util._python3", "allmydata.util.spans", "allmydata.util.statistics", "allmydata.util.time_format", - "allmydata.test.common_py3", ] PORTED_TEST_MODULES = [ "allmydata.test.test_abbreviate", "allmydata.test.test_base32", "allmydata.test.test_base62", + "allmydata.test.test_crypto", "allmydata.test.test_deferredutil", "allmydata.test.test_dictutil", + "allmydata.test.test_happiness", "allmydata.test.test_hashtree", "allmydata.test.test_hashutil", "allmydata.test.test_humanreadable", "allmydata.test.test_iputil", + "allmydata.test.test_log", "allmydata.test.test_netstring", "allmydata.test.test_observer", "allmydata.test.test_pipeline", diff --git a/src/allmydata/util/abbreviate.py b/src/allmydata/util/abbreviate.py index e7bdd8410..f895c3727 100644 --- a/src/allmydata/util/abbreviate.py +++ b/src/allmydata/util/abbreviate.py @@ -10,7 +10,7 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import re from datetime import timedelta diff --git a/src/allmydata/util/assertutil.py b/src/allmydata/util/assertutil.py index af8817702..ed4b8599f 100644 --- a/src/allmydata/util/assertutil.py +++ b/src/allmydata/util/assertutil.py @@ -14,7 +14,7 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 # The API importers expect: diff --git a/src/allmydata/util/base32.py b/src/allmydata/util/base32.py index ba5f80d67..287d214ea 100644 --- a/src/allmydata/util/base32.py +++ b/src/allmydata/util/base32.py @@ -10,7 +10,7 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 if PY2: def backwardscompat_bytes(b): diff --git a/src/allmydata/util/base62.py b/src/allmydata/util/base62.py index 3d80671ba..964baff34 100644 --- a/src/allmydata/util/base62.py +++ b/src/allmydata/util/base62.py @@ -10,7 +10,7 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 if PY2: import string diff --git a/src/allmydata/util/deferredutil.py b/src/allmydata/util/deferredutil.py index e36892f96..9ce05ef3a 100644 --- a/src/allmydata/util/deferredutil.py +++ b/src/allmydata/util/deferredutil.py @@ -11,7 +11,7 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import time diff --git a/src/allmydata/util/dictutil.py b/src/allmydata/util/dictutil.py index d5f0da29b..2a80e7e33 100644 --- a/src/allmydata/util/dictutil.py +++ b/src/allmydata/util/dictutil.py @@ -13,7 +13,7 @@ if PY2: # IMPORTANT: We deliberately don't import dict. The issue is that we're # subclassing dict, so we'd end up exposing Python 3 dict APIs to lots of # code that doesn't support it. - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, list, object, range, str, max, min # noqa: F401 class DictOfSets(dict): diff --git a/src/allmydata/util/gcutil.py b/src/allmydata/util/gcutil.py index 8fb3a64c9..33f1f64f5 100644 --- a/src/allmydata/util/gcutil.py +++ b/src/allmydata/util/gcutil.py @@ -7,7 +7,17 @@ Helpers for managing garbage collection. a result. Register allocation and release of *bare* file descriptors with this object (file objects, socket objects, etc, have their own integration with the garbage collector and don't need to bother with this). + +Ported to Python 3. """ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 __all__ = [ "fileDescriptorResource", diff --git a/src/allmydata/util/hashutil.py b/src/allmydata/util/hashutil.py index d26f5a9b0..96d52c862 100644 --- a/src/allmydata/util/hashutil.py +++ b/src/allmydata/util/hashutil.py @@ -10,7 +10,7 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from past.builtins import chr as byteschr diff --git a/src/allmydata/util/humanreadable.py b/src/allmydata/util/humanreadable.py index 8c7079c3a..60ac57083 100644 --- a/src/allmydata/util/humanreadable.py +++ b/src/allmydata/util/humanreadable.py @@ -11,7 +11,7 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import os from reprlib import Repr diff --git a/src/allmydata/util/iputil.py b/src/allmydata/util/iputil.py index 8754fca53..bd5ea7e78 100644 --- a/src/allmydata/util/iputil.py +++ b/src/allmydata/util/iputil.py @@ -11,7 +11,7 @@ from __future__ import unicode_literals from future.utils import PY2, native_str if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import os, re, socket, subprocess, errno from sys import platform diff --git a/src/allmydata/util/limiter.py b/src/allmydata/util/limiter.py deleted file mode 100644 index 0391ede11..000000000 --- a/src/allmydata/util/limiter.py +++ /dev/null @@ -1,40 +0,0 @@ - -from twisted.internet import defer -from foolscap.api import eventually - -class ConcurrencyLimiter(object): - """I implement a basic concurrency limiter. Add work to it in the form of - (callable, args, kwargs) tuples. No more than LIMIT callables will be - outstanding at any one time. - """ - - def __init__(self, limit=10): - self.limit = limit - self.pending = [] - self.active = 0 - - def __repr__(self): - return "" % (self.active, len(self.pending), - self.limit) - - def add(self, cb, *args, **kwargs): - d = defer.Deferred() - task = (cb, args, kwargs, d) - self.pending.append(task) - self.maybe_start_task() - return d - - def maybe_start_task(self): - if self.active >= self.limit: - return - if not self.pending: - return - (cb, args, kwargs, done_d) = self.pending.pop(0) - self.active += 1 - d = defer.maybeDeferred(cb, *args, **kwargs) - d.addBoth(self._done, done_d) - - def _done(self, res, done_d): - self.active -= 1 - eventually(done_d.callback, res) - eventually(self.maybe_start_task) diff --git a/src/allmydata/util/log.py b/src/allmydata/util/log.py index 454002000..11c78a5a2 100644 --- a/src/allmydata/util/log.py +++ b/src/allmydata/util/log.py @@ -1,4 +1,18 @@ -from allmydata.util import nummedobj +""" +Logging utilities. + +Ported to Python 3. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + +from pyutil import nummedobj from foolscap.logging import log from twisted.python import log as tw_log @@ -36,8 +50,8 @@ class LogMixin(object): def log(self, msg, facility=None, parent=None, *args, **kwargs): if facility is None: facility = self._facility - pmsgid = None - if parent is None: + pmsgid = parent + if pmsgid is None: pmsgid = self._parentmsgid if pmsgid is None: pmsgid = self._grandparentmsgid @@ -54,6 +68,8 @@ class PrefixingLogMixin(nummedobj.NummedObj, LogMixin): LogMixin.__init__(self, facility, grandparentmsgid) if prefix: + if isinstance(prefix, bytes): + prefix = prefix.decode("utf-8", errors="replace") self._prefix = "%s(%s): " % (self.__repr__(), prefix) else: self._prefix = "%s: " % (self.__repr__(),) diff --git a/src/allmydata/util/mathutil.py b/src/allmydata/util/mathutil.py index be88f60b4..42863c30e 100644 --- a/src/allmydata/util/mathutil.py +++ b/src/allmydata/util/mathutil.py @@ -13,7 +13,7 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 # The API importers expect: diff --git a/src/allmydata/util/netstring.py b/src/allmydata/util/netstring.py index 7f2bef377..14e515619 100644 --- a/src/allmydata/util/netstring.py +++ b/src/allmydata/util/netstring.py @@ -10,7 +10,7 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from past.builtins import long diff --git a/src/allmydata/util/nummedobj.py b/src/allmydata/util/nummedobj.py deleted file mode 100644 index 50d7c6454..000000000 --- a/src/allmydata/util/nummedobj.py +++ /dev/null @@ -1,42 +0,0 @@ -import collections, itertools, functools - -objnums = collections.defaultdict(itertools.count) - - -@functools.total_ordering -class NummedObj(object): - """ - This is useful for nicer debug printouts. Instead of objects of the same class being - distinguished from one another by their memory address, they each get a unique number, which - can be read as "the first object of this class", "the second object of this class", etc. This - is especially useful because separate runs of a program will yield identical debug output, - (assuming that the objects get created in the same order in each run). This makes it possible - to diff outputs from separate runs to see what changed, without having to ignore a difference - on every line due to different memory addresses of objects. - """ - - def __init__(self, klass=None): - """ - @param klass: in which class are you counted? If default value of `None', then self.__class__ will be used. - """ - if klass is None: - klass = self.__class__ - self._classname = klass.__name__ - - self._objid = objnums[self._classname].next() - - def __repr__(self): - return "<%s #%d>" % (self._classname, self._objid,) - - def __lt__(self, other): - if isinstance(other, NummedObj): - return (self._objid, self._classname,) < (other._objid, other._classname,) - return NotImplemented - - def __eq__(self, other): - if isinstance(other, NummedObj): - return (self._objid, self._classname,) == (other._objid, other._classname,) - return NotImplemented - - def __hash__(self): - return id(self) diff --git a/src/allmydata/util/observer.py b/src/allmydata/util/observer.py index d5003dfb3..4ebb598c1 100644 --- a/src/allmydata/util/observer.py +++ b/src/allmydata/util/observer.py @@ -11,7 +11,7 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import weakref from twisted.internet import defer diff --git a/src/allmydata/util/pipeline.py b/src/allmydata/util/pipeline.py index df80e2c6c..31f5d5d49 100644 --- a/src/allmydata/util/pipeline.py +++ b/src/allmydata/util/pipeline.py @@ -11,7 +11,7 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from twisted.internet import defer from twisted.python.failure import Failure diff --git a/src/allmydata/util/pollmixin.py b/src/allmydata/util/pollmixin.py index 4eb19b8e3..5d1716853 100644 --- a/src/allmydata/util/pollmixin.py +++ b/src/allmydata/util/pollmixin.py @@ -11,7 +11,7 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import time from twisted.internet import task diff --git a/src/allmydata/util/spans.py b/src/allmydata/util/spans.py index 814904d68..b224f0950 100644 --- a/src/allmydata/util/spans.py +++ b/src/allmydata/util/spans.py @@ -5,7 +5,7 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 class Spans(object): diff --git a/src/allmydata/util/statistics.py b/src/allmydata/util/statistics.py index c5fb5c6bf..a517751d6 100644 --- a/src/allmydata/util/statistics.py +++ b/src/allmydata/util/statistics.py @@ -18,7 +18,7 @@ from __future__ import print_function from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from allmydata.util.mathutil import round_sigfigs import math diff --git a/src/allmydata/util/time_format.py b/src/allmydata/util/time_format.py index 5807791e4..ff267485e 100644 --- a/src/allmydata/util/time_format.py +++ b/src/allmydata/util/time_format.py @@ -11,7 +11,7 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from future.utils import native_str import calendar, datetime, re, time diff --git a/src/allmydata/util/verlib.py b/src/allmydata/util/verlib.py index f69e34e3d..2dfc24a1b 100644 --- a/src/allmydata/util/verlib.py +++ b/src/allmydata/util/verlib.py @@ -11,7 +11,7 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import re diff --git a/src/allmydata/version_checks.py b/src/allmydata/version_checks.py index 51a49d78a..d022055ea 100644 --- a/src/allmydata/version_checks.py +++ b/src/allmydata/version_checks.py @@ -11,7 +11,7 @@ from __future__ import unicode_literals from future.utils import PY2 if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, int, list, object, range, str, max, min # noqa: F401 + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 __all__ = [ "PackagingError", diff --git a/tox.ini b/tox.ini index ab467abd0..0376ab28e 100644 --- a/tox.ini +++ b/tox.ini @@ -49,8 +49,8 @@ commands = tahoe --version [testenv:py36] -# git inside of ratchet.sh needs $HOME. -passenv = HOME +# On macOS, git inside of ratchet.sh needs $HOME. +passenv = {[testenv]passenv} HOME commands = {toxinidir}/misc/python3/ratchet.sh [testenv:integration] @@ -77,6 +77,8 @@ commands = coverage xml [testenv:codechecks] +# On macOS, git inside of towncrier needs $HOME. +passenv = HOME whitelist_externals = /bin/mv commands = @@ -208,7 +210,9 @@ extras = deps = {[testenv]deps} packaging - pyinstaller + # PyInstaller 4.0 drops Python 2 support. When we finish porting to + # Python 3 we can reconsider this constraint. + pyinstaller < 4.0 # Setting PYTHONHASHSEED to a known value assists with reproducible builds. # See https://pyinstaller.readthedocs.io/en/stable/advanced-topics.html#creating-a-reproducible-build setenv=PYTHONHASHSEED=1