From 64516aac410b086df9137e1514464a19c0b975ff Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 28 Sep 2020 10:20:00 -0400 Subject: [PATCH 1/6] Port to Python 3. --- newsfragments/3449.minor | 0 src/allmydata/introducer/interfaces.py | 35 ++++++++++++++++++-------- src/allmydata/test/storage_plugin.py | 4 ++- src/allmydata/util/_python3.py | 1 + 4 files changed, 28 insertions(+), 12 deletions(-) create mode 100644 newsfragments/3449.minor diff --git a/newsfragments/3449.minor b/newsfragments/3449.minor new file mode 100644 index 000000000..e69de29bb diff --git a/src/allmydata/introducer/interfaces.py b/src/allmydata/introducer/interfaces.py index d0ce1fbee..0b7a32fa5 100644 --- a/src/allmydata/introducer/interfaces.py +++ b/src/allmydata/introducer/interfaces.py @@ -1,3 +1,16 @@ +""" +Ported to Python 3. +""" + +from __future__ import unicode_literals +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from future.utils import PY2, native_str +if PY2: + # Omitted types (bytes etc.) so future variants don't confuse Foolscap. + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, object, range, max, min # noqa: F401 from zope.interface import Interface from foolscap.api import StringConstraint, SetOf, DictOf, Any, \ @@ -11,8 +24,8 @@ FURL = StringConstraint(1000) # "app-versions", "my-version", "oldest-supported", and "service-name". # Plus service-specific keys like "anonymous-storage-FURL" and # "permutation-seed-base32" (both for service="storage"). -# * sig_vs (str): "v0-"+base32(signature(msg)) -# * claimed_key_vs (str): "v0-"+base32(pubkey) +# * sig_vs (bytes): "v0-"+base32(signature(msg)) +# * claimed_key_vs (bytes): "v0-"+base32(pubkey) # (nickname, my_version, oldest_supported) refer to the client as a whole. # The my_version/oldest_supported strings can be parsed by an @@ -28,26 +41,26 @@ FURL = StringConstraint(1000) Announcement_v2 = Any() class RIIntroducerSubscriberClient_v2(RemoteInterface): - __remote_name__ = "RIIntroducerSubscriberClient_v2.tahoe.allmydata.com" + __remote_name__ = native_str("RIIntroducerSubscriberClient_v2.tahoe.allmydata.com") def announce_v2(announcements=SetOf(Announcement_v2)): """I accept announcements from the publisher.""" return None -SubscriberInfo = DictOf(str, Any()) +SubscriberInfo = DictOf(bytes, Any()) class RIIntroducerPublisherAndSubscriberService_v2(RemoteInterface): """To publish a service to the world, connect to me and give me your announcement message. I will deliver a copy to all connected subscribers. To hear about services, connect to me and subscribe to a specific service_name.""" - __remote_name__ = "RIIntroducerPublisherAndSubscriberService_v2.tahoe.allmydata.com" + __remote_name__ = native_str("RIIntroducerPublisherAndSubscriberService_v2.tahoe.allmydata.com") def get_version(): - return DictOf(str, Any()) + return DictOf(bytes, Any()) def publish_v2(announcement=Announcement_v2, canary=Referenceable): return None def subscribe_v2(subscriber=RIIntroducerSubscriberClient_v2, - service_name=str, subscriber_info=SubscriberInfo): + service_name=bytes, subscriber_info=SubscriberInfo): """Give me a subscriber reference, and I will call its announce_v2() method with any announcements that match the desired service name. I will ignore duplicate subscriptions. The subscriber_info dictionary @@ -93,11 +106,11 @@ class IIntroducerClient(Interface): version: 0 nickname: unicode app-versions: {} - my-version: str - oldest-supported: str + my-version: bytes + oldest-supported: bytes - service-name: str('storage') - anonymous-storage-FURL: str(furl) + service-name: bytes('storage') + anonymous-storage-FURL: bytes(furl) Note that app-version will be an empty dictionary if either the publishing client or the Introducer are running older code. diff --git a/src/allmydata/test/storage_plugin.py b/src/allmydata/test/storage_plugin.py index 43186122c..52e909b13 100644 --- a/src/allmydata/test/storage_plugin.py +++ b/src/allmydata/test/storage_plugin.py @@ -3,6 +3,8 @@ A storage server plugin the test suite can use to validate the functionality. """ +from future.utils import native_str + from json import ( dumps, ) @@ -36,7 +38,7 @@ from allmydata.client import ( class RIDummy(RemoteInterface): - __remote_name__ = "RIDummy.tahoe.allmydata.com" + __remote_name__ = native_str("RIDummy.tahoe.allmydata.com") def just_some_method(): """ diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index b3fc4c6a4..e9c891aa3 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -34,6 +34,7 @@ PORTED_MODULES = [ "allmydata.hashtree", "allmydata.immutable.happiness_upload", "allmydata.interfaces", + "allmydata.introducer.interfaces", "allmydata.monitor", "allmydata.storage.common", "allmydata.storage.crawler", From 66c6522325b7740ce575917b9a29755bf193d241 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 28 Sep 2020 16:44:50 -0400 Subject: [PATCH 2/6] Unused code. --- src/allmydata/immutable/literal.py | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/src/allmydata/immutable/literal.py b/src/allmydata/immutable/literal.py index 4832da7c1..f5eca564e 100644 --- a/src/allmydata/immutable/literal.py +++ b/src/allmydata/immutable/literal.py @@ -2,7 +2,6 @@ from io import BytesIO from zope.interface import implementer from twisted.internet import defer -from twisted.internet.interfaces import IPushProducer from twisted.protocols import basic from allmydata.interfaces import IImmutableFileNode, ICheckable from allmydata.uri import LiteralFileURI @@ -45,19 +44,6 @@ class _ImmutableFileNodeBase(object): return True -@implementer(IPushProducer) -class LiteralProducer(object): - - def pauseProducing(self): - pass - - def resumeProducing(self): - pass - - def stopProducing(self): - pass - - class LiteralFileNode(_ImmutableFileNodeBase): def __init__(self, filecap): From e3a9f5fa75732c43c8d6845678b838866840f3fe Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 28 Sep 2020 16:49:30 -0400 Subject: [PATCH 3/6] Test and bugfix for LiteralFileNode equality. --- src/allmydata/immutable/literal.py | 9 ++++----- src/allmydata/test/test_immutable.py | 19 +++++++++++++++++++ 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/src/allmydata/immutable/literal.py b/src/allmydata/immutable/literal.py index f5eca564e..59577afb1 100644 --- a/src/allmydata/immutable/literal.py +++ b/src/allmydata/immutable/literal.py @@ -32,16 +32,15 @@ class _ImmutableFileNodeBase(object): def __hash__(self): return self.u.__hash__() + def __eq__(self, other): if isinstance(other, _ImmutableFileNodeBase): - return self.u.__eq__(other.u) + return self.u == other.u else: return False + def __ne__(self, other): - if isinstance(other, _ImmutableFileNodeBase): - return self.u.__eq__(other.u) - else: - return True + return not self == other class LiteralFileNode(_ImmutableFileNodeBase): diff --git a/src/allmydata/test/test_immutable.py b/src/allmydata/test/test_immutable.py index 2646d2c38..12f2012e0 100644 --- a/src/allmydata/test/test_immutable.py +++ b/src/allmydata/test/test_immutable.py @@ -26,6 +26,7 @@ from allmydata.util.consumer import download_to_data from allmydata.interfaces import NotEnoughSharesError from allmydata.immutable.upload import Data from allmydata.immutable.downloader import finder +from allmydata.immutable.literal import LiteralFileNode from .no_network import ( NoNetworkServer, @@ -340,6 +341,24 @@ class Test(GridTestMixin, unittest.TestCase, common.ShouldFailMixin): return d +class LiteralFileNodeTests(unittest.TestCase): + """Tests for LiteralFileNode.""" + + def test_equality(self): + """LiteralFileNodes are equal iff they have the same URI.""" + uri1 = uri.LiteralFileURI(b"1") + uri2 = uri.LiteralFileURI(b"2") + lfn1 = LiteralFileNode(uri1) + lfn1b = LiteralFileNode(uri1) + lfn2 = LiteralFileNode(uri2) + self.assertTrue(lfn1 == lfn1b) + self.assertFalse(lfn1 != lfn1b) + self.assertTrue(lfn1 != lfn2) + self.assertFalse(lfn1 == lfn2) + self.assertTrue(lfn1 != 300) + self.assertFalse(lfn1 == 300) + + # XXX extend these tests to show bad behavior of various kinds from servers: # raising exception from each remove_foo() method, for example From a0a8b4403d4eda7e1cb734a6d5eedb7d9bca5327 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 28 Sep 2020 16:49:51 -0400 Subject: [PATCH 4/6] News file. --- newsfragments/3450.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3450.minor diff --git a/newsfragments/3450.minor b/newsfragments/3450.minor new file mode 100644 index 000000000..e69de29bb From f42634cfe776668a315b93fa750ac44475545840 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 28 Sep 2020 16:51:29 -0400 Subject: [PATCH 5/6] Port to Python 3. --- src/allmydata/immutable/literal.py | 13 +++++++++++++ src/allmydata/util/_python3.py | 1 + 2 files changed, 14 insertions(+) diff --git a/src/allmydata/immutable/literal.py b/src/allmydata/immutable/literal.py index 59577afb1..68db478f3 100644 --- a/src/allmydata/immutable/literal.py +++ b/src/allmydata/immutable/literal.py @@ -1,3 +1,16 @@ +""" +Ported to Python 3. +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + from io import BytesIO from zope.interface import implementer diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 4c5eaa6af..e2b6c3ddb 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -33,6 +33,7 @@ PORTED_MODULES = [ "allmydata.crypto.util", "allmydata.hashtree", "allmydata.immutable.happiness_upload", + "allmydata.immutable.literal", "allmydata.interfaces", "allmydata.monitor", "allmydata.storage.common", From 369cd98d5a1e5d6c786c6a548e02db371d96c2c3 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 29 Sep 2020 10:17:26 -0400 Subject: [PATCH 6/6] Clarify what "JSON-serializable bytes" means. --- src/allmydata/introducer/interfaces.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/allmydata/introducer/interfaces.py b/src/allmydata/introducer/interfaces.py index 0b7a32fa5..9f08f1943 100644 --- a/src/allmydata/introducer/interfaces.py +++ b/src/allmydata/introducer/interfaces.py @@ -112,6 +112,11 @@ class IIntroducerClient(Interface): service-name: bytes('storage') anonymous-storage-FURL: bytes(furl) + In order to be JSON-serializable, all byte strings are assumed to be + ASCII-encoded, and the receiver can therefore decode them into Unicode + strings if they wish. Representation of these fields elsewhere in Tahoe + may differ, e.g. by being unicode strings. + Note that app-version will be an empty dictionary if either the publishing client or the Introducer are running older code. """