Merge pull request #811 from tahoe-lafs/3415.storage-server-python-3

Port allmydata.storage.server to Python 3

Fixes ticket:3415

src/allmydata/storage/server.py

@@ -1,4 +1,18 @@
-from future.utils import bytes_to_native_str
+"""
+Ported to Python 3.
+"""
+from __future__ import division
+from __future__ import absolute_import
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from future.utils import bytes_to_native_str, PY2
+if PY2:
+    # Omit open() to get native behavior where open("w") always accepts native
+    # strings. Omit bytes so we don't leak future's custom bytes.
+    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, pow, round, super, dict, list, object, range, str, max, min  # noqa: F401
+
+
 import os, re, struct, time
 import weakref
 import six
@@ -228,16 +242,18 @@ class StorageServer(service.MultiService, Referenceable):
             # We're on a platform that has no API to get disk stats.
             remaining_space = 2**64
 
-        version = { "http://allmydata.org/tahoe/protocols/storage/v1" :
+        # Unicode strings might be nicer, but for now sticking to bytes since
-                    { "maximum-immutable-share-size": remaining_space,
+        # this is what the wire protocol has always been.
-                      "maximum-mutable-share-size": MAX_MUTABLE_SHARE_SIZE,
+        version = { b"http://allmydata.org/tahoe/protocols/storage/v1" :
-                      "available-space": remaining_space,
+                    { b"maximum-immutable-share-size": remaining_space,
-                      "tolerates-immutable-read-overrun": True,
+                      b"maximum-mutable-share-size": MAX_MUTABLE_SHARE_SIZE,
-                      "delete-mutable-shares-with-zero-length-writev": True,
+                      b"available-space": remaining_space,
-                      "fills-holes-with-zero-bytes": True,
+                      b"tolerates-immutable-read-overrun": True,
-                      "prevents-read-past-end-of-share-data": True,
+                      b"delete-mutable-shares-with-zero-length-writev": True,
+                      b"fills-holes-with-zero-bytes": True,
+                      b"prevents-read-past-end-of-share-data": True,
                       },
-                    "application-version": str(allmydata.__full_version__),
+                    b"application-version": allmydata.__full_version__.encode("utf-8"),
                     }
         return version
 
@@ -671,7 +687,7 @@ class StorageServer(service.MultiService, Referenceable):
                 filename = os.path.join(bucketdir, sharenum_s)
                 msf = MutableShareFile(filename, self)
                 datavs[sharenum] = msf.readv(readv)
-        log.msg("returning shares %s" % (datavs.keys(),),
+        log.msg("returning shares %s" % (list(datavs.keys()),),
                 facility="tahoe.storage", level=log.NOISY, parent=lp)
         self.add_latency("readv", time.time() - start)
         return datavs

src/allmydata/test/common_py3.py

@@ -123,7 +123,7 @@ class ShouldFailMixin(object):
 class ReallyEqualMixin(object):
     def failUnlessReallyEqual(self, a, b, msg=None):
         self.assertEqual(a, b, msg)
-        self.assertEqual(type(a), type(b), "a :: %r, b :: %r, %r" % (a, b, msg))
+        self.assertEqual(type(a), type(b), "a :: %r (%s), b :: %r (%s), %r" % (a, type(a), b, type(b), msg))
 
 
 def skip_if_cannot_represent_filename(u):

src/allmydata/test/test_deepcheck.py

@@ -1,3 +1,4 @@
+from future.utils import native_str
 
 import os, json, urllib
 from twisted.trial import unittest
@@ -945,7 +946,7 @@ class DeepCheckWebBad(DeepCheckBase, unittest.TestCase):
     def _corrupt_some_shares(self, node):
         for (shnum, serverid, sharefile) in self.find_uri_shares(node.get_uri()):
             if shnum in (0,1):
-                yield run_cli("debug", "corrupt-share", sharefile)
+                yield run_cli("debug", "corrupt-share", native_str(sharefile))
 
     def _delete_most_shares(self, node):
         self.delete_shares_numbered(node.get_uri(), range(1,10))

src/allmydata/test/test_storage.py

@@ -366,21 +366,21 @@ class Server(unittest.TestCase):
     def test_declares_fixed_1528(self):
         ss = self.create("test_declares_fixed_1528")
         ver = ss.remote_get_version()
-        sv1 = ver['http://allmydata.org/tahoe/protocols/storage/v1']
+        sv1 = ver[b'http://allmydata.org/tahoe/protocols/storage/v1']
-        self.failUnless(sv1.get('prevents-read-past-end-of-share-data'), sv1)
+        self.failUnless(sv1.get(b'prevents-read-past-end-of-share-data'), sv1)
 
     def test_declares_maximum_share_sizes(self):
         ss = self.create("test_declares_maximum_share_sizes")
         ver = ss.remote_get_version()
-        sv1 = ver['http://allmydata.org/tahoe/protocols/storage/v1']
+        sv1 = ver[b'http://allmydata.org/tahoe/protocols/storage/v1']
-        self.failUnlessIn('maximum-immutable-share-size', sv1)
+        self.failUnlessIn(b'maximum-immutable-share-size', sv1)
-        self.failUnlessIn('maximum-mutable-share-size', sv1)
+        self.failUnlessIn(b'maximum-mutable-share-size', sv1)
 
     def test_declares_available_space(self):
         ss = self.create("test_declares_available_space")
         ver = ss.remote_get_version()
-        sv1 = ver['http://allmydata.org/tahoe/protocols/storage/v1']
+        sv1 = ver[b'http://allmydata.org/tahoe/protocols/storage/v1']
-        self.failUnlessIn('available-space', sv1)
+        self.failUnlessIn(b'available-space', sv1)
 
     def allocate(self, ss, storage_index, sharenums, size, canary=None):
         renew_secret = hashutil.tagged_hash(b"blah", b"%d" % next(self._lease_secret))
@@ -740,6 +740,12 @@ class Server(unittest.TestCase):
         leases = list(ss.get_leases(b"si3"))
         self.failUnlessEqual(len(leases), 2)
 
+    def test_have_shares(self):
+        """By default the StorageServer has no shares."""
+        workdir = self.workdir("test_have_shares")
+        ss = StorageServer(workdir, b"\x00" * 20, readonly_storage=True)
+        self.assertFalse(ss.have_shares())
+
     def test_readonly(self):
         workdir = self.workdir("test_readonly")
         ss = StorageServer(workdir, b"\x00" * 20, readonly_storage=True)
@@ -974,8 +980,8 @@ class MutableServer(unittest.TestCase):
         # Also see if the server explicitly declares that it supports this
         # feature.
         ver = ss.remote_get_version()
-        storage_v1_ver = ver["http://allmydata.org/tahoe/protocols/storage/v1"]
+        storage_v1_ver = ver[b"http://allmydata.org/tahoe/protocols/storage/v1"]
-        self.failUnless(storage_v1_ver.get("fills-holes-with-zero-bytes"))
+        self.failUnless(storage_v1_ver.get(b"fills-holes-with-zero-bytes"))
 
         # If the size is dropped to zero the share is deleted.
         answer = rstaraw(b"si1", secrets,

src/allmydata/util/_python3.py

@@ -41,6 +41,7 @@ PORTED_MODULES = [
     "allmydata.storage.immutable",
     "allmydata.storage.lease",
     "allmydata.storage.mutable",
+    "allmydata.storage.server",
     "allmydata.storage.shares",
     "allmydata.test.common_py3",
     "allmydata.uri",

src/allmydata/util/hashutil.py

@@ -10,7 +10,10 @@ from __future__ import unicode_literals
 
 from future.utils import PY2
 if PY2:
-    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
+    # Don't import bytes to prevent leaking future's bytes.
+    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, dict, list, object, range, str, max, min, bytes as future_bytes  # noqa: F401
+else:
+    future_bytes = bytes
 
 from past.builtins import chr as byteschr
 
@@ -213,7 +216,7 @@ def bucket_cancel_secret_hash(file_cancel_secret, peerid):
 
 
 def _xor(a, b):
-    return b"".join([byteschr(c ^ b) for c in a])
+    return b"".join([byteschr(c ^ b) for c in future_bytes(a)])
 
 
 def hmac(tag, data):