Merge pull request #797 from tahoe-lafs/3396.storage-tests-python-3

Make allmydata.test.test_storage run (but not pass!) on Python 3

Fixes ticket:3396

src/allmydata/client.py

@@ -2,7 +2,10 @@ import os, stat, time, weakref
 from base64 import urlsafe_b64encode
 from functools import partial
 from errno import ENOENT, EPERM
-from ConfigParser import NoSectionError
+try:
+    from ConfigParser import NoSectionError
+except ImportError:
+    from configparser import NoSectionError
 
 from foolscap.furl import (
     decode_furl,

src/allmydata/dirnode.py

@@ -1,4 +1,6 @@
 """Directory Node implementation."""
+from past.builtins import unicode
+
 import time
 
 from zope.interface import implementer
@@ -227,7 +229,7 @@ def pack_children(childrenx, writekey, deep_immutable=False):
     return _pack_normalized_children(children, writekey=writekey, deep_immutable=deep_immutable)
 
 
-ZERO_LEN_NETSTR=netstring('')
+ZERO_LEN_NETSTR=netstring(b'')
 def _pack_normalized_children(children, writekey, deep_immutable=False):
     """Take a dict that maps:
          children[unicode_nfc_name] = (IFileSystemNode, metadata_dict)

src/allmydata/immutable/downloader/fetcher.py

@@ -4,7 +4,7 @@ from foolscap.api import eventually
 from allmydata.interfaces import NotEnoughSharesError, NoSharesError
 from allmydata.util import log
 from allmydata.util.dictutil import DictOfSets
-from common import OVERDUE, COMPLETE, CORRUPT, DEAD, BADSEGNUM, \
+from .common import OVERDUE, COMPLETE, CORRUPT, DEAD, BADSEGNUM, \
      BadSegmentNumberError
 
 class SegmentFetcher(object):

src/allmydata/immutable/downloader/finder.py

@@ -5,7 +5,7 @@ from foolscap.api import eventually
 from allmydata.util import base32, log
 from twisted.internet import reactor
 
-from share import Share, CommonShare
+from .share import Share, CommonShare
 
 def incidentally(res, f, *args, **kwargs):
     """Add me to a Deferred chain like this:

src/allmydata/immutable/downloader/node.py

@@ -13,10 +13,10 @@ from allmydata.hashtree import IncompleteHashTree, BadHashError, \
      NotEnoughHashesError
 
 # local imports
-from finder import ShareFinder
-from fetcher import SegmentFetcher
-from segmentation import Segmentation
-from common import BadCiphertextHashError
+from .finder import ShareFinder
+from .fetcher import SegmentFetcher
+from .segmentation import Segmentation
+from .common import BadCiphertextHashError
 
 class IDownloadStatusHandlingConsumer(Interface):
     def set_download_status_read_event(read_ev):

src/allmydata/immutable/downloader/segmentation.py

@@ -9,7 +9,7 @@ from allmydata.util import log
 from allmydata.util.spans import overlap
 from allmydata.interfaces import DownloadStopped
 
-from common import BadSegmentNumberError, WrongSegmentError
+from .common import BadSegmentNumberError, WrongSegmentError
 
 @implementer(IPushProducer)
 class Segmentation(object):

src/allmydata/immutable/downloader/share.py

@@ -13,7 +13,7 @@ from allmydata.hashtree import IncompleteHashTree, BadHashError, \
 
 from allmydata.immutable.layout import make_write_bucket_proxy
 from allmydata.util.observer import EventStreamObserver
-from common import COMPLETE, CORRUPT, DEAD, BADSEGNUM
+from .common import COMPLETE, CORRUPT, DEAD, BADSEGNUM
 
 
 class LayoutInvalid(Exception):

src/allmydata/immutable/upload.py

@@ -1,3 +1,5 @@
+from past.builtins import long
+
 import os, time, weakref, itertools
 from zope.interface import implementer
 from twisted.python import failure
@@ -26,7 +28,7 @@ from allmydata.interfaces import IUploadable, IUploader, IUploadResults, \
 from allmydata.immutable import layout
 
 from six.moves import cStringIO as StringIO
-from happiness_upload import share_placement, calculate_happiness
+from .happiness_upload import share_placement, calculate_happiness
 
 from ..util.eliotutil import (
     log_call_deferred,

src/allmydata/node.py

@@ -2,12 +2,17 @@
 This module contains classes and functions to implement and manage
 a node for Tahoe-LAFS.
 """
+from past.builtins import unicode
+
 import datetime
 import os.path
 import re
 import types
 import errno
-import ConfigParser
+try:
+    import ConfigParser
+except ImportError:
+    import configparser as ConfigParser
 import tempfile
 from io import BytesIO
 from base64 import b32decode, b32encode
@@ -67,7 +72,7 @@ def _common_valid_config():
 
 # Add our application versions to the data that Foolscap's LogPublisher
 # reports.
-for thing, things_version in get_package_versions().iteritems():
+for thing, things_version in get_package_versions().items():
     app_versions.add_version(thing, str(things_version))
 
 # group 1 will be addr (dotted quad string), group 3 if any will be portnum (string)
@@ -272,7 +277,10 @@ class _Config(object):
         self.config = configparser
 
         nickname_utf8 = self.get_config("node", "nickname", "<unspecified>")
-        self.nickname = nickname_utf8.decode("utf-8")
+        if isinstance(nickname_utf8, bytes):  # Python 2
+            self.nickname = nickname_utf8.decode("utf-8")
+        else:
+            self.nickname = nickname_utf8
         assert type(self.nickname) is unicode
 
     def validate(self, valid_config_sections):

src/allmydata/storage_client.py

@@ -30,9 +30,12 @@ the foolscap-based server implemented in src/allmydata/storage/*.py .
 
 
 import re, time, hashlib
-from ConfigParser import (
-    NoSectionError,
-)
+try:
+    from ConfigParser import (
+        NoSectionError,
+    )
+except ImportError:
+    from configparser import NoSectionError
 import attr
 from zope.interface import (
     Attribute,
@@ -534,11 +537,11 @@ class _NullStorage(object):
     which we can't communicate.
     """
     nickname = ""
-    permutation_seed = hashlib.sha256("").digest()
-    tubid = hashlib.sha256("").digest()
+    permutation_seed = hashlib.sha256(b"").digest()
+    tubid = hashlib.sha256(b"").digest()
     storage_server = None
 
-    lease_seed = hashlib.sha256("").digest()
+    lease_seed = hashlib.sha256(b"").digest()
 
     name = "<unsupported>"
     longname = "<storage with unsupported protocol>"

src/allmydata/test/common.py

@@ -52,7 +52,6 @@ from twisted.internet.defer import inlineCallbacks, returnValue
 from twisted.internet.interfaces import IPullProducer
 from twisted.python import failure
 from twisted.python.filepath import FilePath
-from twisted.application import service
 from twisted.web.error import Error as WebError
 from twisted.internet.interfaces import (
     IStreamServerEndpointStringParser,
@@ -88,6 +87,8 @@ from ..crypto import (
 from .eliotutil import (
     EliotLoggedRunTest,
 )
+# Backwards compatibility imports:
+from .common_py3 import LoggingServiceParent, ShouldFailMixin  # noqa: F401
 
 
 TEST_RSA_KEY_SIZE = 522
@@ -780,53 +781,8 @@ def create_mutable_filenode(contents, mdmf=False, all_contents=None):
     return filenode
 
 
-class LoggingServiceParent(service.MultiService):
-    def log(self, *args, **kwargs):
-        return log.msg(*args, **kwargs)
-
-
 TEST_DATA="\x02"*(Uploader.URI_LIT_SIZE_THRESHOLD+1)
 
-class ShouldFailMixin(object):
-    def shouldFail(self, expected_failure, which, substring,
-                   callable, *args, **kwargs):
-        """Assert that a function call raises some exception. This is a
-        Deferred-friendly version of TestCase.assertRaises() .
-
-        Suppose you want to verify the following function:
-
-         def broken(a, b, c):
-             if a < 0:
-                 raise TypeError('a must not be negative')
-             return defer.succeed(b+c)
-
-        You can use:
-            d = self.shouldFail(TypeError, 'test name',
-                                'a must not be negative',
-                                broken, -4, 5, c=12)
-        in your test method. The 'test name' string will be included in the
-        error message, if any, because Deferred chains frequently make it
-        difficult to tell which assertion was tripped.
-
-        The substring= argument, if not None, must appear in the 'repr'
-        of the message wrapped by this Failure, or the test will fail.
-        """
-
-        assert substring is None or isinstance(substring, str)
-        d = defer.maybeDeferred(callable, *args, **kwargs)
-        def done(res):
-            if isinstance(res, failure.Failure):
-                res.trap(expected_failure)
-                if substring:
-                    message = repr(res.value.args[0])
-                    self.failUnless(substring in message,
-                                    "%s: substring '%s' not in '%s'"
-                                    % (which, substring, message))
-            else:
-                self.fail("%s was supposed to raise %s, not get '%s'" %
-                          (which, expected_failure, res))
-        d.addBoth(done)
-        return d
 
 class WebErrorMixin(object):
     def explain_web_error(self, f):

src/allmydata/test/common_py3.py

@@ -19,11 +19,13 @@ import time
 import signal
 
 from twisted.internet import defer, reactor
+from twisted.application import service
 from twisted.python import failure
 from twisted.trial import unittest
 
 from ..util.assertutil import precondition
 from ..util.encodingutil import unicode_platform, get_filesystem_encoding
+from ..util import log
 
 
 class TimezoneMixin(object):
@@ -77,6 +79,28 @@ class ShouldFailMixin(object):
 
     def shouldFail(self, expected_failure, which, substring,
                    callable, *args, **kwargs):
+        """Assert that a function call raises some exception. This is a
+        Deferred-friendly version of TestCase.assertRaises() .
+
+        Suppose you want to verify the following function:
+
+         def broken(a, b, c):
+             if a < 0:
+                 raise TypeError('a must not be negative')
+             return defer.succeed(b+c)
+
+        You can use:
+            d = self.shouldFail(TypeError, 'test name',
+                                'a must not be negative',
+                                broken, -4, 5, c=12)
+        in your test method. The 'test name' string will be included in the
+        error message, if any, because Deferred chains frequently make it
+        difficult to tell which assertion was tripped.
+
+        The substring= argument, if not None, must appear in the 'repr'
+        of the message wrapped by this Failure, or the test will fail.
+        """
+
         assert substring is None or isinstance(substring, (bytes, unicode))
         d = defer.maybeDeferred(callable, *args, **kwargs)
         def done(res):
@@ -135,3 +159,9 @@ class FakeCanary(object):
         if self.ignore:
             return
         del self.disconnectors[marker]
+
+
+class LoggingServiceParent(service.MultiService):
+    def log(self, *args, **kwargs):
+        return log.msg(*args, **kwargs)
+

src/allmydata/test/eliotutil.py

@@ -2,6 +2,8 @@
 Tools aimed at the interaction between tests and Eliot.
 """
 
+from past.builtins import unicode
+
 __all__ = [
     "RUN_TEST",
     "EliotLoggedRunTest",

src/allmydata/test/test_storage.py

@@ -30,12 +30,11 @@ from allmydata.mutable.layout import MDMFSlotWriteProxy, MDMFSlotReadProxy, \
                                      VERIFICATION_KEY_SIZE, \
                                      SHARE_HASH_CHAIN_SIZE
 from allmydata.interfaces import BadWriteEnablerError
-from allmydata.test.common import LoggingServiceParent, ShouldFailMixin
 from allmydata.test.no_network import NoNetworkServer
 from allmydata.storage_client import (
     _StorageServer,
 )
-from .common_py3 import FakeCanary
+from .common_py3 import FakeCanary, LoggingServiceParent, ShouldFailMixin
 
 
 class FakeStatsProvider(object):

src/allmydata/test/test_system.py

@@ -980,6 +980,8 @@ class CountingDataUploadable(upload.Data):
 
 class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase):
 
+    timeout = 180
+
     def test_connections(self):
         self.basedir = "system/SystemTest/test_connections"
         d = self.set_up_nodes()