Merge remote-tracking branch 'origin/master' into 3458.callremote-unicode

Makefile

@@ -19,8 +19,10 @@ PYTHON=python
 export PYTHON
 PYFLAKES=flake8
 export PYFLAKES
+VIRTUAL_ENV=./.tox/py27
 SOURCES=src/allmydata static misc setup.py
 APPNAME=tahoe-lafs
+TEST_SUITE=allmydata
 
 
 # Top-level, phony targets
@@ -45,6 +47,17 @@ test: .tox/create-venvs.log
 	tox --develop -e codechecks
 # Run all the test environments in parallel to reduce run-time
 	tox --develop -p auto -e 'py27,py36,pypy27'
+.PHONY: test-venv-coverage
+## Run all tests with coverage collection and reporting.
+test-venv-coverage:
+# Special handling for reporting coverage even when the test run fails
+	test_exit=
+	$(VIRTUAL_ENV)/bin/coverage run -m twisted.trial --rterrors --reporter=timing \
+		$(TEST_SUITE) || test_exit="$$?"
+	$(VIRTUAL_ENV)/bin/coverage combine
+	$(VIRTUAL_ENV)/bin/coverage xml || true
+	$(VIRTUAL_ENV)/bin/coverage report
+	if [ ! -z "$$test_exit" ]; then exit "$$test_exit"; fi
 .PHONY: test-py3-all
 ## Run all tests under Python 3
 test-py3-all: .tox/create-venvs.log

misc/python3/Makefile

@@ -0,0 +1,53 @@
+# BBB: Python 3 porting targets
+#
+# NOTE: this Makefile requires GNU make
+
+### Defensive settings for make:
+#     https://tech.davis-hansson.com/p/make/
+SHELL := bash
+.ONESHELL:
+.SHELLFLAGS := -xeu -o pipefail -c
+.SILENT:
+.DELETE_ON_ERROR:
+MAKEFLAGS += --warn-undefined-variables
+MAKEFLAGS += --no-builtin-rules
+
+
+# Top-level, phony targets
+
+.PHONY: default
+default:
+	@echo "no default target"
+
+.PHONY: test-py3-all-before
+## Log the output of running all tests under Python 3 before changes
+test-py3-all-before: ../../.tox/make-test-py3-all-old.log
+.PHONY: test-py3-all-diff
+## Compare the output of running all tests under Python 3 after changes
+test-py3-all-diff: ../../.tox/make-test-py3-all.diff
+
+
+# Real targets
+
+# Gauge the impact of changes on Python 3 compatibility
+# Compare the output from running all tests under Python 3 before and after changes.
+# Before changes:
+# `$ rm -f .tox/make-test-py3-all-*.log && make .tox/make-test-py3-all-old.log`
+# After changes:
+# `$ make .tox/make-test-py3-all.diff`
+$(foreach side,old new,../../.tox/make-test-py3-all-$(side).log):
+	cd "../../"
+	tox --develop --notest -e py36-coverage
+	(make VIRTUAL_ENV=./.tox/py36-coverage TEST_SUITE=allmydata \
+		test-venv-coverage || true) | \
+		sed -E 's/\([0-9]+\.[0-9]{3} secs\)/(#.### secs)/' | \
+		tee "./misc/python3/$(@)"
+../../.tox/make-test-py3-all.diff: ../../.tox/make-test-py3-all-new.log
+	(diff -u "$(<:%-new.log=%-old.log)" "$(<)" || true) | tee "$(@)"
+
+# Locate modules that are candidates for naively converting `unicode` -> `str`.
+# List all Python source files that reference `unicode` but don't reference `str`
+../../.tox/py3-unicode-no-str.ls:
+	cd "../../"
+	find src -type f -iname '*.py' -exec grep -l -E '\Wunicode\W' '{}' ';' | \
+		xargs grep -L '\Wstr\W' | xargs ls -ld | tee "./misc/python3/$(@)"

newsfragments/3448.minor

@@ -0,0 +1 @@
+Convert modules that only reference `unicode` to use `str`.

src/allmydata/scripts/common.py

@@ -2,9 +2,16 @@ from __future__ import print_function
 
 import os, sys, urllib, textwrap
 import codecs
-from six.moves.configparser import NoSectionError
 from os.path import join
+
+# BBB: Python 2 compatibility
+from future.utils import PY2
+if PY2:
+    from future.builtins import str  # noqa: F401
+from six.moves.configparser import NoSectionError
+
 from twisted.python import usage
+
 from allmydata.util.assertutil import precondition
 from allmydata.util.encodingutil import unicode_to_url, quote_output, \
     quote_local_unicode_path, argv_to_abspath
@@ -188,7 +195,7 @@ def get_alias(aliases, path_unicode, default):
     and default is not found in aliases, an UnknownAliasError is
     raised.
     """
-    precondition(isinstance(path_unicode, unicode), path_unicode)
+    precondition(isinstance(path_unicode, str), path_unicode)
 
     from allmydata import uri
     path = path_unicode.encode('utf-8').strip(" ")

src/allmydata/scripts/stats_gatherer.py

@@ -1,7 +1,14 @@
 from __future__ import print_function
 
 import os
+
+# BBB: Python 2 compatibility
+from future.utils import PY2
+if PY2:
+    from future.builtins import str  # noqa: F401
+
 from twisted.python import usage
+
 from allmydata.scripts.common import NoDefaultBasedirOptions
 from allmydata.scripts.create_node import write_tac
 from allmydata.util.assertutil import precondition
@@ -62,7 +69,7 @@ def create_stats_gatherer(config):
     err = config.stderr
     basedir = config['basedir']
     # This should always be called with an absolute Unicode basedir.
-    precondition(isinstance(basedir, unicode), basedir)
+    precondition(isinstance(basedir, str), basedir)
 
     if os.path.exists(basedir):
         if listdir_unicode(basedir):

src/allmydata/scripts/tahoe_check.py

@@ -2,7 +2,14 @@ from __future__ import print_function
 
 import urllib
 import json
+
+# BBB: Python 2 compatibility
+from future.utils import PY2
+if PY2:
+    from future.builtins import str  # noqa: F401
+
 from twisted.protocols.basic import LineOnlyReceiver
+
 from allmydata.scripts.common import get_alias, DEFAULT_ALIAS, escape_path, \
                                      UnknownAliasError
 from allmydata.scripts.common_http import do_http, format_http_error
@@ -101,7 +108,7 @@ def check_location(options, where):
 
 def check(options):
     if len(options.locations) == 0:
-        errno = check_location(options, unicode())
+        errno = check_location(options, str())
         if errno != 0:
             return errno
         return 0
@@ -325,7 +332,7 @@ class DeepCheckStreamer(LineOnlyReceiver, object):
 
     def run(self, options):
         if len(options.locations) == 0:
-            errno = self.deepcheck_location(options, unicode())
+            errno = self.deepcheck_location(options, str())
             if errno != 0:
                 return errno
             return 0

src/allmydata/stats.py

@@ -1,13 +1,16 @@
 from __future__ import print_function
 
-from past.builtins import unicode
-
 import json
 import os
 import pprint
 import time
 from collections import deque
 
+# BBB: Python 2 compatibility
+from future.utils import PY2
+if PY2:
+    from future.builtins import str  # noqa: F401
+
 from twisted.internet import reactor
 from twisted.application import service
 from twisted.application.internet import TimerService
@@ -157,7 +160,7 @@ class StatsProvider(Referenceable, service.MultiService):
         service.MultiService.startService(self)
 
     def count(self, name, delta=1):
-        if isinstance(name, unicode):
+        if isinstance(name, str):
             name = name.encode("utf-8")
         val = self.counters.setdefault(name, 0)
         self.counters[name] = val + delta
@@ -178,7 +181,7 @@ class StatsProvider(Referenceable, service.MultiService):
         def to_bytes(d):
             result = {}
             for (k, v) in d.items():
-                if isinstance(k, unicode):
+                if isinstance(k, str):
                     k = k.encode("utf-8")
                 result[k] = v
             return result

src/allmydata/test/check_load.py

@@ -37,6 +37,11 @@ a mean of 10kB and a max of 100MB, so filesize=min(int(1.0/random(.0002)),1e8)
 import os, sys, httplib, binascii
 import urllib, json, random, time, urlparse
 
+# BBB: Python 2 compatibility
+from future.utils import PY2
+if PY2:
+    from future.builtins import str  # noqa: F401
+
 if sys.argv[1] == "--stats":
     statsfiles = sys.argv[2:]
     # gather stats every 10 seconds, do a moving-window average of the last
@@ -116,7 +121,7 @@ def listdir(nodeurl, root, remote_pathname):
     assert nodetype == "dirnode"
     global directories_read
     directories_read += 1
-    children = dict( [(unicode(name),value)
+    children = dict( [(str(name),value)
                       for (name,value)
                       in d["children"].iteritems()] )
     return children

src/allmydata/test/check_memory.py

@@ -1,18 +1,25 @@
 from __future__ import print_function
 
 import os, shutil, sys, urllib, time, stat, urlparse
+
+# BBB: Python 2 compatibility
+from future.utils import PY2
+if PY2:
+    from future.builtins import str  # noqa: F401
 from six.moves import cStringIO as StringIO
+
 from twisted.internet import defer, reactor, protocol, error
 from twisted.application import service, internet
 from twisted.web import client as tw_client
+from twisted.python import log, procutils
+from foolscap.api import Tub, fireEventually, flushEventualQueue
+
 from allmydata import client, introducer
 from allmydata.immutable import upload
 from allmydata.scripts import create_node
 from allmydata.util import fileutil, pollmixin
 from allmydata.util.fileutil import abspath_expanduser_unicode
 from allmydata.util.encodingutil import get_filesystem_encoding
-from foolscap.api import Tub, fireEventually, flushEventualQueue
-from twisted.python import log, procutils
 
 class StallableHTTPGetterDiscarder(tw_client.HTTPPageGetter, object):
     full_speed_ahead = False
@@ -69,7 +76,7 @@ class SystemFramework(pollmixin.PollMixin):
     numnodes = 7
 
     def __init__(self, basedir, mode):
-        self.basedir = basedir = abspath_expanduser_unicode(unicode(basedir))
+        self.basedir = basedir = abspath_expanduser_unicode(str(basedir))
         if not (basedir + os.path.sep).startswith(abspath_expanduser_unicode(u".") + os.path.sep):
             raise AssertionError("safety issue: basedir must be a subdir")
         self.testdir = testdir = os.path.join(basedir, "test")

src/allmydata/test/common.py

@@ -46,6 +46,7 @@ from testtools.twistedsupport import (
     flush_logged_errors,
 )
 
+from twisted.application import service
 from twisted.plugin import IPlugin
 from twisted.internet import defer
 from twisted.internet.defer import inlineCallbacks, returnValue
@@ -87,8 +88,7 @@ from ..crypto import (
 from .eliotutil import (
     EliotLoggedRunTest,
 )
-# Backwards compatibility imports:
+from .common_util import ShouldFailMixin  # noqa: F401
-from .common_py3 import LoggingServiceParent, ShouldFailMixin  # noqa: F401
 
 
 TEST_RSA_KEY_SIZE = 522
@@ -781,6 +781,11 @@ def create_mutable_filenode(contents, mdmf=False, all_contents=None):
     return filenode
 
 
+class LoggingServiceParent(service.MultiService):
+    def log(self, *args, **kwargs):
+        return log.msg(*args, **kwargs)
+
+
 TEST_DATA=b"\x02"*(Uploader.URI_LIT_SIZE_THRESHOLD+1)
 
 

src/allmydata/test/common_py3.py

@@ -1,170 +0,0 @@
-"""
-Common utilities that have been ported to Python 3.
-
-Ported to Python 3.
-"""
-
-from __future__ import unicode_literals
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-from future.utils import PY2
-if PY2:
-    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
-from past.builtins import unicode
-
-import os
-import time
-import signal
-
-from twisted.internet import defer, reactor
-from twisted.application import service
-from twisted.python import failure
-from twisted.trial import unittest
-
-from ..util.assertutil import precondition
-from ..util.encodingutil import unicode_platform, get_filesystem_encoding
-from ..util import log
-
-
-class TimezoneMixin(object):
-
-    def setTimezone(self, timezone):
-        def tzset_if_possible():
-            # Windows doesn't have time.tzset().
-            if hasattr(time, 'tzset'):
-                time.tzset()
-
-        unset = object()
-        originalTimezone = os.environ.get('TZ', unset)
-        def restoreTimezone():
-            if originalTimezone is unset:
-                del os.environ['TZ']
-            else:
-                os.environ['TZ'] = originalTimezone
-            tzset_if_possible()
-
-        os.environ['TZ'] = timezone
-        self.addCleanup(restoreTimezone)
-        tzset_if_possible()
-
-    def have_working_tzset(self):
-        return hasattr(time, 'tzset')
-
-
-class SignalMixin(object):
-    # This class is necessary for any code which wants to use Processes
-    # outside the usual reactor.run() environment. It is copied from
-    # Twisted's twisted.test.test_process . Note that Twisted-8.2.0 uses
-    # something rather different.
-    sigchldHandler = None
-
-    def setUp(self):
-        # make sure SIGCHLD handler is installed, as it should be on
-        # reactor.run(). problem is reactor may not have been run when this
-        # test runs.
-        if hasattr(reactor, "_handleSigchld") and hasattr(signal, "SIGCHLD"):
-            self.sigchldHandler = signal.signal(signal.SIGCHLD,
-                                                reactor._handleSigchld)
-        return super(SignalMixin, self).setUp()
-
-    def tearDown(self):
-        if self.sigchldHandler:
-            signal.signal(signal.SIGCHLD, self.sigchldHandler)
-        return super(SignalMixin, self).tearDown()
-
-
-class ShouldFailMixin(object):
-
-    def shouldFail(self, expected_failure, which, substring,
-                   callable, *args, **kwargs):
-        """Assert that a function call raises some exception. This is a
-        Deferred-friendly version of TestCase.assertRaises() .
-
-        Suppose you want to verify the following function:
-
-         def broken(a, b, c):
-             if a < 0:
-                 raise TypeError('a must not be negative')
-             return defer.succeed(b+c)
-
-        You can use:
-            d = self.shouldFail(TypeError, 'test name',
-                                'a must not be negative',
-                                broken, -4, 5, c=12)
-        in your test method. The 'test name' string will be included in the
-        error message, if any, because Deferred chains frequently make it
-        difficult to tell which assertion was tripped.
-
-        The substring= argument, if not None, must appear in the 'repr'
-        of the message wrapped by this Failure, or the test will fail.
-        """
-
-        assert substring is None or isinstance(substring, (bytes, unicode))
-        d = defer.maybeDeferred(callable, *args, **kwargs)
-        def done(res):
-            if isinstance(res, failure.Failure):
-                res.trap(expected_failure)
-                if substring:
-                    self.failUnless(substring in str(res),
-                                    "%s: substring '%s' not in '%s'"
-                                    % (which, substring, str(res)))
-                # return the Failure for further analysis, but in a form that
-                # doesn't make the Deferred chain think that we failed.
-                return [res]
-            else:
-                self.fail("%s was supposed to raise %s, not get '%s'" %
-                          (which, expected_failure, res))
-        d.addBoth(done)
-        return d
-
-
-class ReallyEqualMixin(object):
-    def failUnlessReallyEqual(self, a, b, msg=None):
-        self.assertEqual(a, b, msg)
-        self.assertEqual(type(a), type(b), "a :: %r (%s), b :: %r (%s), %r" % (a, type(a), b, type(b), msg))
-
-
-def skip_if_cannot_represent_filename(u):
-    precondition(isinstance(u, unicode))
-
-    enc = get_filesystem_encoding()
-    if not unicode_platform():
-        try:
-            u.encode(enc)
-        except UnicodeEncodeError:
-            raise unittest.SkipTest("A non-ASCII filename could not be encoded on this platform.")
-
-
-class Marker(object):
-    pass
-
-class FakeCanary(object):
-    """For use in storage tests.
-
-    Can be moved back to test_storage.py once enough Python 3 porting has been
-    done.
-    """
-    def __init__(self, ignore_disconnectors=False):
-        self.ignore = ignore_disconnectors
-        self.disconnectors = {}
-    def notifyOnDisconnect(self, f, *args, **kwargs):
-        if self.ignore:
-            return
-        m = Marker()
-        self.disconnectors[m] = (f, args, kwargs)
-        return m
-    def dontNotifyOnDisconnect(self, marker):
-        if self.ignore:
-            return
-        del self.disconnectors[marker]
-    def getRemoteTubID(self):
-        return None
-    def getPeer(self):
-        return "<fake>"
-
-
-class LoggingServiceParent(service.MultiService):
-    def log(self, *args, **kwargs):
-        return log.msg(*args, **kwargs)

src/allmydata/test/common_util.py

@@ -1,22 +1,33 @@
 from __future__ import print_function
 
 import os
+import time
+import signal
 from random import randrange
 from six.moves import StringIO
 
 from twisted.internet import reactor, defer
+from twisted.python import failure
 from twisted.trial import unittest
 
 from ..util.assertutil import precondition
 from ..scripts import runner
-from allmydata.util.encodingutil import get_io_encoding
+from allmydata.util.encodingutil import unicode_platform, get_filesystem_encoding, get_io_encoding
 # Imported for backwards compatibility:
 from future.utils import bord, bchr, binary_type
-from .common_py3 import (
+from past.builtins import unicode
-    SignalMixin, skip_if_cannot_represent_filename, ReallyEqualMixin, ShouldFailMixin
-)
 
 
+def skip_if_cannot_represent_filename(u):
+    precondition(isinstance(u, unicode))
+
+    enc = get_filesystem_encoding()
+    if not unicode_platform():
+        try:
+            u.encode(enc)
+        except UnicodeEncodeError:
+            raise unittest.SkipTest("A non-ASCII filename could not be encoded on this platform.")
+
 def skip_if_cannot_represent_argv(u):
     precondition(isinstance(u, unicode))
     try:
@@ -78,6 +89,34 @@ def flip_one_bit(s, offset=0, size=None):
     return result
 
 
+class ReallyEqualMixin(object):
+    def failUnlessReallyEqual(self, a, b, msg=None):
+        self.assertEqual(a, b, msg)
+        self.assertEqual(type(a), type(b), "a :: %r (%s), b :: %r (%s), %r" % (a, type(a), b, type(b), msg))
+
+
+class SignalMixin(object):
+    # This class is necessary for any code which wants to use Processes
+    # outside the usual reactor.run() environment. It is copied from
+    # Twisted's twisted.test.test_process . Note that Twisted-8.2.0 uses
+    # something rather different.
+    sigchldHandler = None
+
+    def setUp(self):
+        # make sure SIGCHLD handler is installed, as it should be on
+        # reactor.run(). problem is reactor may not have been run when this
+        # test runs.
+        if hasattr(reactor, "_handleSigchld") and hasattr(signal, "SIGCHLD"):
+            self.sigchldHandler = signal.signal(signal.SIGCHLD,
+                                                reactor._handleSigchld)
+        return super(SignalMixin, self).setUp()
+
+    def tearDown(self):
+        if self.sigchldHandler:
+            signal.signal(signal.SIGCHLD, self.sigchldHandler)
+        return super(SignalMixin, self).tearDown()
+
+
 class StallMixin(object):
     def stall(self, res=None, delay=1):
         d = defer.Deferred()
@@ -85,6 +124,79 @@ class StallMixin(object):
         return d
 
 
+class Marker(object):
+    pass
+
+class FakeCanary(object):
+    """For use in storage tests.
+
+    Can be moved back to test_storage.py once enough Python 3 porting has been
+    done.
+    """
+    def __init__(self, ignore_disconnectors=False):
+        self.ignore = ignore_disconnectors
+        self.disconnectors = {}
+    def notifyOnDisconnect(self, f, *args, **kwargs):
+        if self.ignore:
+            return
+        m = Marker()
+        self.disconnectors[m] = (f, args, kwargs)
+        return m
+    def dontNotifyOnDisconnect(self, marker):
+        if self.ignore:
+            return
+        del self.disconnectors[marker]
+    def getRemoteTubID(self):
+        return None
+    def getPeer(self):
+        return "<fake>"
+
+
+class ShouldFailMixin(object):
+
+    def shouldFail(self, expected_failure, which, substring,
+                   callable, *args, **kwargs):
+        """Assert that a function call raises some exception. This is a
+        Deferred-friendly version of TestCase.assertRaises() .
+
+        Suppose you want to verify the following function:
+
+         def broken(a, b, c):
+             if a < 0:
+                 raise TypeError('a must not be negative')
+             return defer.succeed(b+c)
+
+        You can use:
+            d = self.shouldFail(TypeError, 'test name',
+                                'a must not be negative',
+                                broken, -4, 5, c=12)
+        in your test method. The 'test name' string will be included in the
+        error message, if any, because Deferred chains frequently make it
+        difficult to tell which assertion was tripped.
+
+        The substring= argument, if not None, must appear in the 'repr'
+        of the message wrapped by this Failure, or the test will fail.
+        """
+
+        assert substring is None or isinstance(substring, (bytes, unicode))
+        d = defer.maybeDeferred(callable, *args, **kwargs)
+        def done(res):
+            if isinstance(res, failure.Failure):
+                res.trap(expected_failure)
+                if substring:
+                    self.failUnless(substring in str(res),
+                                    "%s: substring '%s' not in '%s'"
+                                    % (which, substring, str(res)))
+                # return the Failure for further analysis, but in a form that
+                # doesn't make the Deferred chain think that we failed.
+                return [res]
+            else:
+                self.fail("%s was supposed to raise %s, not get '%s'" %
+                          (which, expected_failure, res))
+        d.addBoth(done)
+        return d
+
+
 class TestMixin(SignalMixin):
     def setUp(self):
         return super(TestMixin, self).setUp()
@@ -132,6 +244,31 @@ class TestMixin(SignalMixin):
             self.fail("Reactor was still active when it was required to be quiescent.")
 
 
+class TimezoneMixin(object):
+
+    def setTimezone(self, timezone):
+        def tzset_if_possible():
+            # Windows doesn't have time.tzset().
+            if hasattr(time, 'tzset'):
+                time.tzset()
+
+        unset = object()
+        originalTimezone = os.environ.get('TZ', unset)
+        def restoreTimezone():
+            if originalTimezone is unset:
+                del os.environ['TZ']
+            else:
+                os.environ['TZ'] = originalTimezone
+            tzset_if_possible()
+
+        os.environ['TZ'] = timezone
+        self.addCleanup(restoreTimezone)
+        tzset_if_possible()
+
+    def have_working_tzset(self):
+        return hasattr(time, 'tzset')
+
+
 try:
     import win32file
     import win32con

src/allmydata/test/eliotutil.py

@@ -2,7 +2,10 @@
 Tools aimed at the interaction between tests and Eliot.
 """
 
-from past.builtins import unicode
+# BBB: Python 2 compatibility
+# Can't use `builtins.str` because it's not JSON encodable:
+# `exceptions.TypeError: <class 'future.types.newstr.newstr'> is not JSON-encodeable`
+from past.builtins import unicode as str
 
 __all__ = [
     "RUN_TEST",
@@ -29,7 +32,7 @@ from twisted.internet.defer import (
 
 _NAME = Field.for_types(
     u"name",
-    [unicode],
+    [str],
     u"The name of the test.",
 )
 

src/allmydata/test/mutable/test_version.py

@@ -1,9 +1,16 @@
 from __future__ import print_function
 
 import os
+
+# BBB: Python 2 compatibility
+from future.utils import PY2
+if PY2:
+    from future.builtins import str  # noqa: F401
 from six.moves import cStringIO as StringIO
+
 from twisted.internet import defer
 from twisted.trial import unittest
+
 from allmydata import uri
 from allmydata.interfaces import SDMF_VERSION, MDMF_VERSION
 from allmydata.util import base32, consumer, mathutil
@@ -75,7 +82,7 @@ class Version(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin, \
             fso = debug.FindSharesOptions()
             storage_index = base32.b2a(n.get_storage_index())
             fso.si_s = storage_index
-            fso.nodedirs = [os.path.dirname(abspath_expanduser_unicode(unicode(storedir)))
+            fso.nodedirs = [os.path.dirname(abspath_expanduser_unicode(str(storedir)))
                             for (i,ss,storedir)
                             in self.iterate_servers()]
             fso.stdout = StringIO()

src/allmydata/test/test_auth.py

@@ -1,3 +1,8 @@
+# BBB: Python 2 compatibility
+from future.utils import PY2
+if PY2:
+    from future.builtins import str  # noqa: F401
+
 from twisted.trial import unittest
 from twisted.python import filepath
 from twisted.cred import error, credentials
@@ -39,7 +44,7 @@ class AccountFileCheckerKeyTests(unittest.TestCase):
     def setUp(self):
         self.account_file = filepath.FilePath(self.mktemp())
         self.account_file.setContent(DUMMY_ACCOUNTS)
-        abspath = abspath_expanduser_unicode(unicode(self.account_file.path))
+        abspath = abspath_expanduser_unicode(str(self.account_file.path))
         self.checker = auth.AccountFileChecker(None, abspath)
 
     def test_unknown_user(self):

src/allmydata/test/test_crawler.py

@@ -27,8 +27,7 @@ from allmydata.util import fileutil, hashutil, pollmixin
 from allmydata.storage.server import StorageServer, si_b2a
 from allmydata.storage.crawler import ShareCrawler, TimeSliceExceeded
 
-from allmydata.test.common_py3 import FakeCanary
+from allmydata.test.common_util import StallMixin, FakeCanary
-from allmydata.test.common_util import StallMixin
 
 class BucketEnumeratingCrawler(ShareCrawler):
     cpu_slice = 500 # make sure it can complete in a single slice

src/allmydata/test/test_deepcheck.py

@@ -1,9 +1,22 @@
+import os, json, urllib
+
+# BBB: Python 2 compatibility
+# Can't use `builtins.str` because something deep in Twisted callbacks ends up repr'ing
+# a `future.types.newstr.newstr` as a *Python 3* byte string representation under
+# *Python 2*:
+#   File "/home/rpatterson/src/work/sfu/tahoe-lafs/.tox/py27/lib/python2.7/site-packages/allmydata/util/netstring.py", line 43, in split_netstring
+#    assert data[position] == b","[0], position
+# exceptions.AssertionError: 15
+# ...
+# (Pdb) pp data
+# '334:12:b\'mutable-good\',90:URI:SSK-RO:...
+from past.builtins import unicode as str
 from future.utils import native_str
 
-import os, json, urllib
 from twisted.trial import unittest
 from twisted.internet import defer
 from twisted.internet.defer import inlineCallbacks, returnValue
+
 from allmydata.immutable import upload
 from allmydata.mutable.common import UnrecoverableFileError
 from allmydata.mutable.publish import MutableData
@@ -917,13 +930,13 @@ class DeepCheckWebBad(DeepCheckBase, unittest.TestCase):
         if nodetype == "mutable":
             mutable_uploadable = MutableData("mutable file contents")
             d = self.g.clients[0].create_mutable_file(mutable_uploadable)
-            d.addCallback(lambda n: self.root.set_node(unicode(name), n))
+            d.addCallback(lambda n: self.root.set_node(str(name), n))
         elif nodetype == "large":
             large = upload.Data("Lots of data\n" * 1000 + name + "\n", None)
-            d = self.root.add_file(unicode(name), large)
+            d = self.root.add_file(str(name), large)
         elif nodetype == "small":
             small = upload.Data("Small enough for a LIT", None)
-            d = self.root.add_file(unicode(name), small)
+            d = self.root.add_file(str(name), small)
 
         d.addCallback(self._stash_node, name)
 

src/allmydata/test/test_encodingutil.py

@@ -77,7 +77,7 @@ from twisted.trial import unittest
 
 from twisted.python.filepath import FilePath
 
-from allmydata.test.common_py3 import (
+from allmydata.test.common_util import (
     ReallyEqualMixin, skip_if_cannot_represent_filename,
 )
 from allmydata.util import encodingutil, fileutil

src/allmydata/test/test_happiness.py

@@ -23,7 +23,7 @@ from hypothesis.strategies import text, sets
 from allmydata.immutable import happiness_upload
 from allmydata.util.happinessutil import servers_of_happiness, \
     shares_by_server, merge_servers
-from allmydata.test.common_py3 import ShouldFailMixin
+from allmydata.test.common import ShouldFailMixin
 
 
 class HappinessUploadUtils(unittest.TestCase):

src/allmydata/test/test_iputil.py

@@ -23,7 +23,7 @@ from tenacity import retry, stop_after_attempt
 from foolscap.api import Tub
 
 from allmydata.util import iputil, gcutil
-import allmydata.test.common_py3 as testutil
+import allmydata.test.common_util as testutil
 from allmydata.util.namespace import Namespace
 
 

src/allmydata/test/test_storage.py

@@ -51,7 +51,8 @@ from allmydata.test.no_network import NoNetworkServer
 from allmydata.storage_client import (
     _StorageServer,
 )
-from .common_py3 import FakeCanary, LoggingServiceParent, ShouldFailMixin
+from .common import LoggingServiceParent, ShouldFailMixin
+from .common_util import FakeCanary
 
 
 class UtilTests(unittest.TestCase):

src/allmydata/test/test_storage_web.py

@@ -50,7 +50,7 @@ from allmydata.web.storage import (
     StorageStatusElement,
     remove_prefix
 )
-from .common_py3 import FakeCanary
+from .common_util import FakeCanary
 
 def remove_tags(s):
     s = re.sub(br'<[^>]*>', b' ', s)

src/allmydata/test/test_time_format.py

@@ -16,7 +16,7 @@ import time
 
 from twisted.trial import unittest
 
-from allmydata.test.common_py3 import TimezoneMixin
+from allmydata.test.common_util import TimezoneMixin
 from allmydata.util import time_format
 
 

src/allmydata/test/test_upload.py

@@ -28,12 +28,12 @@ from allmydata.util import log, base32
 from allmydata.util.assertutil import precondition
 from allmydata.util.deferredutil import DeferredListShouldSucceed
 from allmydata.test.no_network import GridTestMixin
-from allmydata.test.common_py3 import ShouldFailMixin
 from allmydata.storage_client import StorageFarmBroker
 from allmydata.storage.server import storage_index_to_dir
 from allmydata.client import _Client
 from .common import (
     EMPTY_CLIENT_CONFIG,
+    ShouldFailMixin,
 )
 from functools import reduce
 

src/allmydata/test/web/test_introducer.py

@@ -33,7 +33,7 @@ from .common import (
 from ..common import (
     SameProcessStreamEndpointAssigner,
 )
-from ..common_py3 import (
+from ..common_util import (
     FakeCanary,
 )
 from ..common_web import (

src/allmydata/test/web/test_web.py

@@ -52,7 +52,7 @@ from allmydata.interfaces import (
 )
 from allmydata.mutable import servermap, publish, retrieve
 from .. import common_util as testutil
-from ..common_py3 import TimezoneMixin
+from ..common_util import TimezoneMixin
 from ..common_web import (
     do_http,
     Error,

src/allmydata/util/_python3.py

@@ -54,7 +54,6 @@ PORTED_MODULES = [
     "allmydata.storage.mutable",
     "allmydata.storage.server",
     "allmydata.storage.shares",
-    "allmydata.test.common_py3",
     "allmydata.test.no_network",
     "allmydata.uri",
     "allmydata.util._python3",