Merge branch '3409.allmydata-storage-immutable-python-3' into 3415.storage-server-python-3

src/allmydata/stats.py

@@ -1,5 +1,7 @@
 from __future__ import print_function
 
+from past.builtins import unicode
+
 import json
 import os
 import pprint
@@ -155,6 +157,8 @@ class StatsProvider(Referenceable, service.MultiService):
         service.MultiService.startService(self)
 
     def count(self, name, delta=1):
+        if isinstance(name, unicode):
+            name = name.encode("utf-8")
         val = self.counters.setdefault(name, 0)
         self.counters[name] = val + delta
 
@@ -170,7 +174,18 @@ class StatsProvider(Referenceable, service.MultiService):
         return ret
 
     def remote_get_stats(self):
-        return self.get_stats()
+        # The remote API expects keys to be bytes:
+        def to_bytes(d):
+            result = {}
+            for (k, v) in d.items():
+                if isinstance(k, unicode):
+                    k = k.encode("utf-8")
+                result[k] = v
+            return result
+
+        stats = self.get_stats()
+        return {b"counters": to_bytes(stats["counters"]),
+                b"stats": to_bytes(stats["stats"])}
 
     def _connected(self, gatherer, nickname):
         gatherer.callRemoteOnly('provide', self, nickname or '')

src/allmydata/storage/immutable.py

@@ -1,4 +1,15 @@
-from future.utils import bytes_to_native_str
+"""
+Ported to Python 3.
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from future.utils import PY2, bytes_to_native_str
+if PY2:
+    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
 import os, stat, struct, time
 

src/allmydata/test/test_storage.py

@@ -30,7 +30,7 @@ from allmydata import interfaces
 from allmydata.util import fileutil, hashutil, base32
 from allmydata.storage.server import StorageServer
 from allmydata.storage.mutable import MutableShareFile
-from allmydata.storage.immutable import BucketWriter, BucketReader
+from allmydata.storage.immutable import BucketWriter, BucketReader, ShareFile
 from allmydata.storage.common import DataTooLargeError, storage_index_to_dir, \
      UnknownMutableContainerVersionError, UnknownImmutableContainerVersionError
 from allmydata.storage.lease import LeaseInfo
@@ -2974,3 +2974,38 @@ class Stats(unittest.TestCase):
         self.failUnless(output["get"]["95_0_percentile"] is None, output)
         self.failUnless(output["get"]["99_0_percentile"] is None, output)
         self.failUnless(output["get"]["99_9_percentile"] is None, output)
+
+
+class ShareFileTests(unittest.TestCase):
+    """Tests for allmydata.storage.immutable.ShareFile."""
+
+    def get_sharefile(self):
+        sf = ShareFile(self.mktemp(), max_size=1000, create=True)
+        sf.write_share_data(0, b"abc")
+        sf.write_share_data(2, b"DEF")
+        # Should be b'abDEF' now.
+        return sf
+
+    def test_read_write(self):
+        """Basic writes can be read."""
+        sf = self.get_sharefile()
+        self.assertEqual(sf.read_share_data(0, 3), b"abD")
+        self.assertEqual(sf.read_share_data(1, 4), b"bDEF")
+
+    def test_reads_beyond_file_end(self):
+        """Reads beyond the file size are truncated."""
+        sf = self.get_sharefile()
+        self.assertEqual(sf.read_share_data(0, 10), b"abDEF")
+        self.assertEqual(sf.read_share_data(5, 10), b"")
+
+    def test_too_large_write(self):
+        """Can't do write larger than file size."""
+        sf = self.get_sharefile()
+        with self.assertRaises(DataTooLargeError):
+            sf.write_share_data(0, b"x" * 3000)
+
+    def test_no_leases_cancelled(self):
+        """If no leases were cancelled, IndexError is raised."""
+        sf = self.get_sharefile()
+        with self.assertRaises(IndexError):
+            sf.cancel_lease(b"garbage")

src/allmydata/util/_python3.py

@@ -37,6 +37,7 @@ PORTED_MODULES = [
     "allmydata.monitor",
     "allmydata.storage.crawler",
     "allmydata.storage.expirer",
+    "allmydata.storage.immutable",
     "allmydata.storage.server",
     "allmydata.test.common_py3",
     "allmydata.uri",