Merge remote-tracking branch 'origin/master' into 3453.downloader-share-python-3

src/allmydata/immutable/downloader/__init__.py

@@ -0,0 +1,13 @@
+"""
+Ported to Python 3.
+"""
+
+from __future__ import unicode_literals
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from future.utils import PY2
+if PY2:
+    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
+

src/allmydata/immutable/downloader/common.py

@@ -1,3 +1,15 @@
+"""
+Ported to Python 3.
+"""
+
+from __future__ import unicode_literals
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from future.utils import PY2
+if PY2:
+    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
 (AVAILABLE, PENDING, OVERDUE, COMPLETE, CORRUPT, DEAD, BADSEGNUM) = \
  ("AVAILABLE", "PENDING", "OVERDUE", "COMPLETE", "CORRUPT", "DEAD", "BADSEGNUM")

src/allmydata/immutable/downloader/fetcher.py

@@ -1,4 +1,15 @@
+"""
+Ported to Python 3.
+"""
+
 from __future__ import print_function
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from future.utils import PY2
+if PY2:
+    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
 from twisted.python.failure import Failure
 from foolscap.api import eventually
@@ -222,7 +233,7 @@ class SegmentFetcher(object):
             # add_shares() or no_more_shares() later.
 
     def _cancel_all_requests(self):
-        for o in self._share_observers.values():
+        for o in list(self._share_observers.values()):
             o.cancel()
         self._share_observers = {}
 

src/allmydata/immutable/downloader/finder.py

@@ -1,3 +1,14 @@
+"""
+Ported to Python 3.
+"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from future.utils import PY2
+if PY2:
+    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
 import time
 now = time.time

src/allmydata/immutable/downloader/node.py

@@ -1,3 +1,14 @@
+"""
+Ported to Python 3.
+"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from future.utils import PY2
+if PY2:
+    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
 import time
 now = time.time

src/allmydata/immutable/downloader/segmentation.py

@@ -1,3 +1,14 @@
+"""
+Ported to Python 3.
+"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from future.utils import PY2
+if PY2:
+    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
 import time
 now = time.time

src/allmydata/immutable/downloader/status.py

@@ -1,3 +1,14 @@
+"""
+Ported to Python 3.
+"""
+from __future__ import division
+from __future__ import absolute_import
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from future.utils import PY2
+if PY2:
+    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
 import itertools
 from zope.interface import implementer
@@ -9,8 +20,8 @@ class ReadEvent(object):
         self._ev = ev
         self._ds = ds
 
-    def update(self, bytes, decrypttime, pausetime):
+    def update(self, bytes_returned, decrypttime, pausetime):
-        self._ev["bytes_returned"] += bytes
+        self._ev["bytes_returned"] += bytes_returned
         self._ev["decrypt_time"] += decrypttime
         self._ev["paused_time"] += pausetime
 
@@ -257,7 +268,7 @@ class DownloadStatus(object):
             # else ignore completed requests
         if not total_outstanding:
             return 1.0
-        return 1.0 * total_received / total_outstanding
+        return total_received / total_outstanding
 
     def using_helper(self):
         return False

src/allmydata/immutable/literal.py

@@ -1,8 +1,20 @@
+"""
+Ported to Python 3.
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from future.utils import PY2
+if PY2:
+    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
+
 from io import BytesIO
 
 from zope.interface import implementer
 from twisted.internet import defer
-from twisted.internet.interfaces import IPushProducer
 from twisted.protocols import basic
 from allmydata.interfaces import IImmutableFileNode, ICheckable
 from allmydata.uri import LiteralFileURI
@@ -33,29 +45,15 @@ class _ImmutableFileNodeBase(object):
 
     def __hash__(self):
         return self.u.__hash__()
+
     def __eq__(self, other):
         if isinstance(other, _ImmutableFileNodeBase):
-            return self.u.__eq__(other.u)
+            return self.u == other.u
         else:
             return False
+
     def __ne__(self, other):
-        if isinstance(other, _ImmutableFileNodeBase):
+        return not self == other
-            return self.u.__eq__(other.u)
-        else:
-            return True
-
-
-@implementer(IPushProducer)
-class LiteralProducer(object):
-
-    def pauseProducing(self):
-        pass
-
-    def resumeProducing(self):
-        pass
-
-    def stopProducing(self):
-        pass
 
 
 class LiteralFileNode(_ImmutableFileNodeBase):

src/allmydata/test/test_download.py

@@ -1334,6 +1334,21 @@ class FakeNode(object):
 
 
 class Selection(unittest.TestCase):
+    def test_failure(self):
+        """If the fetch loop fails, it tell the Node the fetch failed."""
+        node = FakeNode()
+        # Simulate a failure:
+        node.get_num_segments = lambda: 1/0
+        sf = SegmentFetcher(node, 0, 3, None)
+        sf.add_shares([])
+        d = flushEventualQueue()
+        def _check1(ign):
+            [_] = self.flushLoggedErrors(ZeroDivisionError)
+            self.failUnless(node.failed)
+            self.failUnless(node.failed.check(ZeroDivisionError))
+        d.addCallback(_check1)
+        return d
+
     def test_no_shares(self):
         node = FakeNode()
         sf = SegmentFetcher(node, 0, 3, None)

src/allmydata/test/test_immutable.py

@@ -26,6 +26,7 @@ from allmydata.util.consumer import download_to_data
 from allmydata.interfaces import NotEnoughSharesError
 from allmydata.immutable.upload import Data
 from allmydata.immutable.downloader import finder
+from allmydata.immutable.literal import LiteralFileNode
 
 from .no_network import (
     NoNetworkServer,
@@ -340,6 +341,24 @@ class Test(GridTestMixin, unittest.TestCase, common.ShouldFailMixin):
         return d
 
 
+class LiteralFileNodeTests(unittest.TestCase):
+    """Tests for LiteralFileNode."""
+
+    def test_equality(self):
+        """LiteralFileNodes are equal iff they have the same URI."""
+        uri1 = uri.LiteralFileURI(b"1")
+        uri2 = uri.LiteralFileURI(b"2")
+        lfn1 = LiteralFileNode(uri1)
+        lfn1b = LiteralFileNode(uri1)
+        lfn2 = LiteralFileNode(uri2)
+        self.assertTrue(lfn1 == lfn1b)
+        self.assertFalse(lfn1 != lfn1b)
+        self.assertTrue(lfn1 != lfn2)
+        self.assertFalse(lfn1 == lfn2)
+        self.assertTrue(lfn1 != 300)
+        self.assertFalse(lfn1 == 300)
+
+
 # XXX extend these tests to show bad behavior of various kinds from servers:
 # raising exception from each remove_foo() method, for example
 

src/allmydata/util/_python3.py

@@ -32,8 +32,16 @@ PORTED_MODULES = [
     "allmydata.crypto.rsa",
     "allmydata.crypto.util",
     "allmydata.hashtree",
-    "allmydata.immutable.happiness_upload",
+    "allmydata.immutable.downloader",
+    "allmydata.immutable.downloader.common",
+    "allmydata.immutable.downloader.fetcher",
+    "allmydata.immutable.downloader.finder",
+    "allmydata.immutable.downloader.node",
+    "allmydata.immutable.downloader.segmentation",
     "allmydata.immutable.downloader.share",
+    "allmydata.immutable.downloader.status",
+    "allmydata.immutable.happiness_upload",
+    "allmydata.immutable.literal",
     "allmydata.interfaces",
     "allmydata.introducer.interfaces",
     "allmydata.monitor",