Merge remote-tracking branch 'origin/master' into 3472.windows-coverage-reports

src/allmydata/mutable/filenode.py

@@ -147,9 +147,9 @@ class MutableFileNode(object):
 
     def _get_initial_contents(self, contents):
         if contents is None:
-            return MutableData("")
+            return MutableData(b"")
 
-        if isinstance(contents, str):
+        if isinstance(contents, bytes):
             return MutableData(contents)
 
         if IMutableUploadable.providedBy(contents):
@@ -884,9 +884,9 @@ class MutableFileVersion(object):
         d = self._try_to_download_data()
         def _apply(old_contents):
             new_contents = modifier(old_contents, self._servermap, first_time)
-            precondition((isinstance(new_contents, str) or
+            precondition((isinstance(new_contents, bytes) or
                           new_contents is None),
-                         "Modifier function must return a string "
+                         "Modifier function must return bytes "
                          "or None")
 
             if new_contents is None or new_contents == old_contents:
@@ -960,7 +960,7 @@ class MutableFileVersion(object):
         c = consumer.MemoryConsumer()
         # modify will almost certainly write, so we need the privkey.
         d = self._read(c, fetch_privkey=True)
-        d.addCallback(lambda mc: "".join(mc.chunks))
+        d.addCallback(lambda mc: b"".join(mc.chunks))
         return d
 
 
@@ -1076,7 +1076,7 @@ class MutableFileVersion(object):
             start = offset
             rest = offset + data.get_size()
             new = old[:start]
-            new += "".join(data.read(data.get_size()))
+            new += b"".join(data.read(data.get_size()))
             new += old[rest:]
             return new
         return self._modify(m, None)

src/allmydata/mutable/publish.py

@@ -1,5 +1,5 @@
 import os, time
-from six.moves import cStringIO as StringIO
+from io import BytesIO
 from itertools import count
 from zope.interface import implementer
 from twisted.internet import defer
@@ -46,7 +46,7 @@ class PublishStatus(object):
         self.size = None
         self.status = "Not started"
         self.progress = 0.0
-        self.counter = self.statusid_counter.next()
+        self.counter = next(self.statusid_counter)
         self.started = time.time()
 
     def add_per_server_time(self, server, elapsed):
@@ -305,7 +305,7 @@ class Publish(object):
         # Our update process fetched these for us. We need to update
         # them in place as publishing happens.
         self.blockhashes = {} # (shnum, [blochashes])
-        for (i, bht) in blockhashes.iteritems():
+        for (i, bht) in list(blockhashes.items()):
             # We need to extract the leaves from our old hash tree.
             old_segcount = mathutil.div_ceil(version[4],
                                              version[3])
@@ -313,7 +313,7 @@ class Publish(object):
             bht = dict(enumerate(bht))
             h.set_hashes(bht)
             leaves = h[h.get_leaf_index(0):]
-            for j in xrange(self.num_segments - len(leaves)):
+            for j in range(self.num_segments - len(leaves)):
                 leaves.append(None)
 
             assert len(leaves) >= self.num_segments
@@ -509,10 +509,10 @@ class Publish(object):
         # This will eventually hold the block hash chain for each share
         # that we publish. We define it this way so that empty publishes
         # will still have something to write to the remote slot.
-        self.blockhashes = dict([(i, []) for i in xrange(self.total_shares)])
+        self.blockhashes = dict([(i, []) for i in range(self.total_shares)])
-        for i in xrange(self.total_shares):
+        for i in range(self.total_shares):
             blocks = self.blockhashes[i]
-            for j in xrange(self.num_segments):
+            for j in range(self.num_segments):
                 blocks.append(None)
         self.sharehash_leaves = None # eventually [sharehashes]
         self.sharehashes = {} # shnum -> [sharehash leaves necessary to
@@ -526,7 +526,7 @@ class Publish(object):
         return self.done_deferred
 
     def _get_some_writer(self):
-        return list(self.writers.values()[0])[0]
+        return list(list(self.writers.values())[0])[0]
 
     def _update_status(self):
         self._status.set_status("Sending Shares: %d placed out of %d, "
@@ -684,7 +684,7 @@ class Publish(object):
         salt = os.urandom(16)
         assert self._version == SDMF_VERSION
 
-        for shnum, writers in self.writers.iteritems():
+        for shnum, writers in self.writers.items():
             for writer in writers:
                 writer.put_salt(salt)
 
@@ -704,7 +704,7 @@ class Publish(object):
         self.log("Pushing segment %d of %d" % (segnum + 1, self.num_segments))
         data = self.data.read(segsize)
         # XXX: This is dumb. Why return a list?
-        data = "".join(data)
+        data = b"".join(data)
 
         assert len(data) == segsize, len(data)
 
@@ -732,7 +732,7 @@ class Publish(object):
         for i in range(len(crypttext_pieces)):
             offset = i * piece_size
             piece = crypttext[offset:offset+piece_size]
-            piece = piece + "\x00"*(piece_size - len(piece)) # padding
+            piece = piece + b"\x00"*(piece_size - len(piece)) # padding
             crypttext_pieces[i] = piece
             assert len(piece) == piece_size
         d = fec.encode(crypttext_pieces)
@@ -751,7 +751,7 @@ class Publish(object):
         results, salt = encoded_and_salt
         shares, shareids = results
         self._status.set_status("Pushing segment")
-        for i in xrange(len(shares)):
+        for i in range(len(shares)):
             sharedata = shares[i]
             shareid = shareids[i]
             if self._version == MDMF_VERSION:
@@ -786,7 +786,7 @@ class Publish(object):
     def push_encprivkey(self):
         encprivkey = self._encprivkey
         self._status.set_status("Pushing encrypted private key")
-        for shnum, writers in self.writers.iteritems():
+        for shnum, writers in self.writers.items():
             for writer in writers:
                 writer.put_encprivkey(encprivkey)
 
@@ -794,7 +794,7 @@ class Publish(object):
     def push_blockhashes(self):
         self.sharehash_leaves = [None] * len(self.blockhashes)
         self._status.set_status("Building and pushing block hash tree")
-        for shnum, blockhashes in self.blockhashes.iteritems():
+        for shnum, blockhashes in list(self.blockhashes.items()):
             t = hashtree.HashTree(blockhashes)
             self.blockhashes[shnum] = list(t)
             # set the leaf for future use.
@@ -808,7 +808,7 @@ class Publish(object):
     def push_sharehashes(self):
         self._status.set_status("Building and pushing share hash chain")
         share_hash_tree = hashtree.HashTree(self.sharehash_leaves)
-        for shnum in xrange(len(self.sharehash_leaves)):
+        for shnum in range(len(self.sharehash_leaves)):
             needed_indices = share_hash_tree.needed_hashes(shnum)
             self.sharehashes[shnum] = dict( [ (i, share_hash_tree[i])
                                              for i in needed_indices] )
@@ -824,7 +824,7 @@ class Publish(object):
         #   - Get the checkstring of the resulting layout; sign that.
         #   - Push the signature
         self._status.set_status("Pushing root hashes and signature")
-        for shnum in xrange(self.total_shares):
+        for shnum in range(self.total_shares):
             writers = self.writers[shnum]
             for writer in writers:
                 writer.put_root_hash(self.root_hash)
@@ -852,7 +852,7 @@ class Publish(object):
         signable = self._get_some_writer().get_signable()
         self.signature = rsa.sign_data(self._privkey, signable)
 
-        for (shnum, writers) in self.writers.iteritems():
+        for (shnum, writers) in self.writers.items():
             for writer in writers:
                 writer.put_signature(self.signature)
         self._status.timings['sign'] = time.time() - started
@@ -867,7 +867,7 @@ class Publish(object):
         ds = []
         verification_key = rsa.der_string_from_verifying_key(self._pubkey)
 
-        for (shnum, writers) in self.writers.copy().iteritems():
+        for (shnum, writers) in list(self.writers.copy().items()):
             for writer in writers:
                 writer.put_verification_key(verification_key)
                 self.num_outstanding += 1
@@ -1003,7 +1003,7 @@ class Publish(object):
 
         # TODO: Precompute this.
         shares = []
-        for shnum, writers in self.writers.iteritems():
+        for shnum, writers in self.writers.items():
             shares.extend([x.shnum for x in writers if x.server == server])
         known_shnums = set(shares)
         surprise_shares -= known_shnums
@@ -1198,7 +1198,7 @@ class Publish(object):
 class MutableFileHandle(object):
     """
     I am a mutable uploadable built around a filehandle-like object,
-    usually either a StringIO instance or a handle to an actual file.
+    usually either a BytesIO instance or a handle to an actual file.
     """
 
     def __init__(self, filehandle):
@@ -1268,14 +1268,14 @@ class MutableFileHandle(object):
 class MutableData(MutableFileHandle):
     """
     I am a mutable uploadable built around a string, which I then cast
-    into a StringIO and treat as a filehandle.
+    into a BytesIO and treat as a filehandle.
     """
 
     def __init__(self, s):
         # Take a string and return a file-like uploadable.
-        assert isinstance(s, str)
+        assert isinstance(s, bytes)
 
-        MutableFileHandle.__init__(self, StringIO(s))
+        MutableFileHandle.__init__(self, BytesIO(s))
 
 
 @implementer(IMutableUploadable)
@@ -1361,7 +1361,7 @@ class TransformingUploadable(object):
 
         self.log("reading %d bytes of new data" % length)
         new_data = self._newdata.read(length)
-        new_data = "".join(new_data)
+        new_data = b"".join(new_data)
 
         self._read_marker += len(old_start_data + new_data + old_end_data)
 

src/allmydata/test/mutable/test_datahandle.py

@@ -1,17 +1,29 @@
+"""
+Ported to Python 3.
+"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from future.utils import PY2
+if PY2:
+    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
+
 from twisted.trial import unittest
 from allmydata.mutable.publish import MutableData
 
 class DataHandle(unittest.TestCase):
     def setUp(self):
-        self.test_data = "Test Data" * 50000
+        self.test_data = b"Test Data" * 50000
         self.uploadable = MutableData(self.test_data)
 
 
     def test_datahandle_read(self):
         chunk_size = 10
-        for i in xrange(0, len(self.test_data), chunk_size):
+        for i in range(0, len(self.test_data), chunk_size):
             data = self.uploadable.read(chunk_size)
-            data = "".join(data)
+            data = b"".join(data)
             start = i
             end = i + chunk_size
             self.failUnlessEqual(data, self.test_data[start:end])
@@ -28,7 +40,7 @@ class DataHandle(unittest.TestCase):
         # disturbing the location of the seek pointer.
         chunk_size = 100
         data = self.uploadable.read(chunk_size)
-        self.failUnlessEqual("".join(data), self.test_data[:chunk_size])
+        self.failUnlessEqual(b"".join(data), self.test_data[:chunk_size])
 
         # Now get the size.
         size = self.uploadable.get_size()
@@ -38,4 +50,4 @@ class DataHandle(unittest.TestCase):
         more_data = self.uploadable.read(chunk_size)
         start = chunk_size
         end = chunk_size * 2
-        self.failUnlessEqual("".join(more_data), self.test_data[start:end])
+        self.failUnlessEqual(b"".join(more_data), self.test_data[start:end])

src/allmydata/test/mutable/test_different_encoding.py

@@ -1,3 +1,15 @@
+"""
+Ported to Python 3.
+"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from future.utils import PY2
+if PY2:
+    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
+
 from twisted.trial import unittest
 from .util import FakeStorage, make_nodemaker
 
@@ -10,7 +22,7 @@ class DifferentEncoding(unittest.TestCase):
         # create a file with 3-of-20, then modify it with a client configured
         # to do 3-of-10. #1510 tracks a failure here
         self.nodemaker.default_encoding_parameters["n"] = 20
-        d = self.nodemaker.create_mutable_file("old contents")
+        d = self.nodemaker.create_mutable_file(b"old contents")
         def _created(n):
             filecap = n.get_cap().to_string()
             del n # we want a new object, not the cached one
@@ -19,6 +31,6 @@ class DifferentEncoding(unittest.TestCase):
             return n2
         d.addCallback(_created)
         def modifier(old_contents, servermap, first_time):
-            return "new contents"
+            return b"new contents"
         d.addCallback(lambda n: n.modify(modifier))
         return d

src/allmydata/test/mutable/test_filehandle.py

@@ -1,21 +1,33 @@
+"""
+Ported to Python 3.
+"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from future.utils import PY2
+if PY2:
+    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
+
 import os
-from six.moves import cStringIO as StringIO
+from io import BytesIO
 from twisted.trial import unittest
 from allmydata.mutable.publish import MutableFileHandle
 
 class FileHandle(unittest.TestCase):
     def setUp(self):
-        self.test_data = "Test Data" * 50000
+        self.test_data = b"Test Data" * 50000
-        self.sio = StringIO(self.test_data)
+        self.sio = BytesIO(self.test_data)
         self.uploadable = MutableFileHandle(self.sio)
 
 
     def test_filehandle_read(self):
         self.basedir = "mutable/FileHandle/test_filehandle_read"
         chunk_size = 10
-        for i in xrange(0, len(self.test_data), chunk_size):
+        for i in range(0, len(self.test_data), chunk_size):
             data = self.uploadable.read(chunk_size)
-            data = "".join(data)
+            data = b"".join(data)
             start = i
             end = i + chunk_size
             self.failUnlessEqual(data, self.test_data[start:end])
@@ -33,7 +45,7 @@ class FileHandle(unittest.TestCase):
         # disturbing the location of the seek pointer.
         chunk_size = 100
         data = self.uploadable.read(chunk_size)
-        self.failUnlessEqual("".join(data), self.test_data[:chunk_size])
+        self.failUnlessEqual(b"".join(data), self.test_data[:chunk_size])
 
         # Now get the size.
         size = self.uploadable.get_size()
@@ -43,26 +55,26 @@ class FileHandle(unittest.TestCase):
         more_data = self.uploadable.read(chunk_size)
         start = chunk_size
         end = chunk_size * 2
-        self.failUnlessEqual("".join(more_data), self.test_data[start:end])
+        self.failUnlessEqual(b"".join(more_data), self.test_data[start:end])
 
 
     def test_filehandle_file(self):
         # Make sure that the MutableFileHandle works on a file as well
-        # as a StringIO object, since in some cases it will be asked to
+        # as a BytesIO object, since in some cases it will be asked to
         # deal with files.
         self.basedir = self.mktemp()
         # necessary? What am I doing wrong here?
         os.mkdir(self.basedir)
         f_path = os.path.join(self.basedir, "test_file")
-        f = open(f_path, "w")
+        f = open(f_path, "wb")
         f.write(self.test_data)
         f.close()
-        f = open(f_path, "r")
+        f = open(f_path, "rb")
 
         uploadable = MutableFileHandle(f)
 
         data = uploadable.read(len(self.test_data))
-        self.failUnlessEqual("".join(data), self.test_data)
+        self.failUnlessEqual(b"".join(data), self.test_data)
         size = uploadable.get_size()
         self.failUnlessEqual(size, len(self.test_data))
 

src/allmydata/test/mutable/util.py

@@ -1,4 +1,6 @@
-from six.moves import cStringIO as StringIO
+from past.builtins import long
+
+from io import BytesIO
 import attr
 from twisted.internet import defer, reactor
 from foolscap.api import eventually, fireEventually
@@ -75,8 +77,8 @@ class FakeStorage(object):
         if peerid not in self._peers:
             self._peers[peerid] = {}
         shares = self._peers[peerid]
-        f = StringIO()
+        f = BytesIO()
-        f.write(shares.get(shnum, ""))
+        f.write(shares.get(shnum, b""))
         f.seek(offset)
         f.write(data)
         shares[shnum] = f.getvalue()
@@ -129,7 +131,7 @@ class FakeStorageServer(object):
         readv = {}
         for shnum, (testv, writev, new_length) in tw_vectors.items():
             for (offset, length, op, specimen) in testv:
-                assert op in ("le", "eq", "ge")
+                assert op in (b"le", b"eq", b"ge")
             # TODO: this isn't right, the read is controlled by read_vector,
             # not by testv
             readv[shnum] = [ specimen
@@ -222,10 +224,10 @@ def make_peer(s, i):
 
     :rtype: ``Peer``
     """
-    peerid = base32.b2a(tagged_hash("peerid", "%d" % i)[:20])
+    peerid = base32.b2a(tagged_hash(b"peerid", b"%d" % i)[:20])
     fss = FakeStorageServer(peerid, s)
     ann = {
-        "anonymous-storage-FURL": "pb://%s@nowhere/fake" % (peerid,),
+        "anonymous-storage-FURL": b"pb://%s@nowhere/fake" % (peerid,),
         "permutation-seed-base32": peerid,
     }
     return Peer(peerid=peerid, storage_server=fss, announcement=ann)
@@ -297,7 +299,7 @@ def make_nodemaker_with_storage_broker(storage_broker, keysize):
 
     :param StorageFarmBroker peers: The storage broker to use.
     """
-    sh = client.SecretHolder("lease secret", "convergence secret")
+    sh = client.SecretHolder(b"lease secret", b"convergence secret")
     keygen = client.KeyGenerator()
     if keysize:
         keygen.set_default_keysize(keysize)

src/allmydata/test/no_network.py

@@ -26,13 +26,19 @@ if PY2:
 from past.builtins import unicode
 
 import os
+from base64 import b32encode
+from functools import (
+    partial,
+)
 from zope.interface import implementer
 from twisted.application import service
 from twisted.internet import defer
 from twisted.python.failure import Failure
 from twisted.web.error import Error
 from foolscap.api import Referenceable, fireEventually, RemoteException
-from base64 import b32encode
+from foolscap.ipb import (
+    IRemoteReference,
+)
 import treq
 
 from allmydata.util.assertutil import _assert
@@ -59,14 +65,29 @@ class IntentionalError(Exception):
 class Marker(object):
     pass
 
+fireNow = partial(defer.succeed, None)
+
+@implementer(IRemoteReference)
 class LocalWrapper(object):
-    def __init__(self, original):
+    """
+    A ``LocalWrapper`` presents the remote reference interface to a local
+    object which implements a ``RemoteInterface``.
+    """
+    def __init__(self, original, fireEventually=fireEventually):
+        """
+        :param Callable[[], Deferred[None]] fireEventually: Get a Deferred
+            that will fire at some point.  This is used to control when
+            ``callRemote`` calls the remote method.  The default value allows
+            the reactor to iterate before the call happens.  Use ``fireNow``
+            to call the remote method synchronously.
+        """
         self.original = original
         self.broken = False
         self.hung_until = None
         self.post_call_notifier = None
         self.disconnectors = {}
         self.counter_by_methname = {}
+        self._fireEventually = fireEventually
 
     def _clear_counters(self):
         self.counter_by_methname = {}
@@ -82,7 +103,7 @@ class LocalWrapper(object):
         # selected return values.
         def wrap(a):
             if isinstance(a, Referenceable):
-                return LocalWrapper(a)
+                return self._wrap(a)
             else:
                 return a
         args = tuple([wrap(a) for a in args])
@@ -110,7 +131,7 @@ class LocalWrapper(object):
                 return d2
             return _really_call()
 
-        d = fireEventually()
+        d = self._fireEventually()
         d.addCallback(lambda res: _call())
         def _wrap_exception(f):
             return Failure(RemoteException(f))
@@ -124,10 +145,10 @@ class LocalWrapper(object):
             if methname == "allocate_buckets":
                 (alreadygot, allocated) = res
                 for shnum in allocated:
-                    allocated[shnum] = LocalWrapper(allocated[shnum])
+                    allocated[shnum] = self._wrap(allocated[shnum])
             if methname == "get_buckets":
                 for shnum in res:
-                    res[shnum] = LocalWrapper(res[shnum])
+                    res[shnum] = self._wrap(res[shnum])
             return res
         d.addCallback(_return_membrane)
         if self.post_call_notifier:
@@ -141,6 +162,10 @@ class LocalWrapper(object):
     def dontNotifyOnDisconnect(self, marker):
         del self.disconnectors[marker]
 
+    def _wrap(self, value):
+        return LocalWrapper(value, self._fireEventually)
+
+
 def wrap_storage_server(original):
     # Much of the upload/download code uses rref.version (which normally
     # comes from rrefutil.add_version_to_remote_reference). To avoid using a

src/allmydata/test/test_python3.py

@@ -8,7 +8,7 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-from future.utils import PY2
+from future.utils import PY2, native_str
 if PY2:
     from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
@@ -44,10 +44,9 @@ class Python3PortingEffortTests(SynchronousTestCase):
                 ),
             ),
         )
-    if PY2:
+    test_finished_porting.todo = native_str(
-        test_finished_porting.skip = "For some reason todo isn't working on Python 2 now"
+        "https://tahoe-lafs.org/trac/tahoe-lafs/milestone/Support%20Python%203 should be completed",
-    else:
+    )
-        test_finished_porting.todo = "https://tahoe-lafs.org/trac/tahoe-lafs/milestone/Support%20Python%203 should be completed"
 
     def test_ported_modules_exist(self):
         """

src/allmydata/test/test_uri.py

@@ -11,7 +11,7 @@ from __future__ import unicode_literals
 
 from future.utils import PY2
 if PY2:
-    from future.builtins import filter, map, zip, ascii, chr, dict, hex, input, next, oct, open, pow, round, super, bytes, int, list, object, range, str, max, min  # noqa: F401
+    from future.builtins import filter, map, zip, ascii, chr, dict, hex, input, next, oct, open, pow, round, super, bytes, list, object, range, str, max, min  # noqa: F401
 
 import os
 from twisted.trial import unittest

src/allmydata/test/test_util.py

@@ -9,6 +9,7 @@ from __future__ import unicode_literals
 
 from future.utils import PY2
 if PY2:
+    # open is not here because we want to use native strings on Py2
     from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 import six
 import os, time, sys

src/allmydata/util/_python3.py

@@ -89,6 +89,9 @@ PORTED_MODULES = [
 ]
 
 PORTED_TEST_MODULES = [
+    "allmydata.test.mutable.test_datahandle",
+    "allmydata.test.mutable.test_different_encoding",
+    "allmydata.test.mutable.test_filehandle",
     "allmydata.test.test_abbreviate",
     "allmydata.test.test_base32",
     "allmydata.test.test_base62",

src/allmydata/util/observer.py

@@ -61,7 +61,7 @@ class OneShotObserverList(object):
 
     def _fire(self, result):
         for w in self._watchers:
-            eventually(w.callback, result)
+            w.callback(result)
         del self._watchers
         self.__repr__ = self._fired_repr