From c3aff634a0a61c4cc4767f61c6fefb0a9f8063f1 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 24 Nov 2020 14:17:52 -0500 Subject: [PATCH 01/22] Add missing attribute docs --- src/allmydata/storage_client.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 2b0ec5af1..823c15b7a 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -154,6 +154,9 @@ class StorageFarmBroker(service.MultiService): I'm also responsible for subscribing to the IntroducerClient to find out about new servers as they are announced by the Introducer. + :ivar _tub_maker: A one-argument callable which accepts a dictionary of + "handler overrides" and returns a ``foolscap.api.Tub``. + :ivar StorageClientConfig storage_client_config: Values from the node configuration file relating to storage behavior. """ From 34b5068f5c7880f1e892394615a21a9d8fa9a4d7 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 24 Nov 2020 14:18:04 -0500 Subject: [PATCH 02/22] Just use a real Tub for this case --- src/allmydata/test/test_storage_client.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/allmydata/test/test_storage_client.py b/src/allmydata/test/test_storage_client.py index fa3a34b15..3086e7491 100644 --- a/src/allmydata/test/test_storage_client.py +++ b/src/allmydata/test/test_storage_client.py @@ -505,11 +505,13 @@ class StoragePluginWebPresence(AsyncTestCase): ) -def make_broker(tub_maker=lambda h: Mock()): +def make_broker(tub_maker=None): """ Create a ``StorageFarmBroker`` with the given tub maker and an empty client configuration. """ + if tub_maker is None: + tub_maker = lambda handler_overrides: Tub() return StorageFarmBroker(True, tub_maker, EMPTY_CLIENT_CONFIG) From 9f4be1fbf03b2a09816ce71ad449479f72d2add7 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 24 Nov 2020 14:23:20 -0500 Subject: [PATCH 03/22] Speed the tests back up --- src/allmydata/test/test_storage_client.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/allmydata/test/test_storage_client.py b/src/allmydata/test/test_storage_client.py index 3086e7491..524d88272 100644 --- a/src/allmydata/test/test_storage_client.py +++ b/src/allmydata/test/test_storage_client.py @@ -511,7 +511,13 @@ def make_broker(tub_maker=None): client configuration. """ if tub_maker is None: - tub_maker = lambda handler_overrides: Tub() + # Use a pre-generated key so the tests don't spend a lot of time + # generating new ones. + data = FilePath(__file__).sibling(b"data") + privkey = data.child(b"node.pem") + tub_maker = lambda handler_overrides: Tub( + certData=privkey.getContent(), + ) return StorageFarmBroker(True, tub_maker, EMPTY_CLIENT_CONFIG) From a78c8056fb22f259fd15719a684fca305400dc65 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 24 Nov 2020 14:51:51 -0500 Subject: [PATCH 04/22] Get rid of the first mock use in `test_threshold_reached` --- src/allmydata/test/data/node.pem | 45 +++++++++++++++++++++ src/allmydata/test/test_storage_client.py | 48 +++++++++++++++++------ 2 files changed, 81 insertions(+), 12 deletions(-) create mode 100644 src/allmydata/test/data/node.pem diff --git a/src/allmydata/test/data/node.pem b/src/allmydata/test/data/node.pem new file mode 100644 index 000000000..dd0928cdd --- /dev/null +++ b/src/allmydata/test/data/node.pem @@ -0,0 +1,45 @@ +-----BEGIN CERTIFICATE----- +MIICojCCAYoCAQEwDQYJKoZIhvcNAQELBQAwFzEVMBMGA1UEAwwMbmV3cGJfdGhp +bmd5MB4XDTIwMDEwMjAxNDAzM1oXDTIxMDEwMTAxNDAzM1owFzEVMBMGA1UEAwwM +bmV3cGJfdGhpbmd5MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA1iNV +z07PYwZwucl87QlL2TFZvDxD4flZ/p3BZE3DCT5Efn9w2NT4sHXL1e+R/qsDFuNG +bw1y1TRM0DGK6Wr0XRT2mLQULNgB8y/HrhcSdONsYRyWdj+LimyECKjwh0iSkApv +Yj/7IOuq6dOoh67YXPdf75OHLShm4+8q8fuwhBL+nuuO4NhZDJKupYHcnuCkcF88 +LN77HKrrgbpyVmeghUkwJMLeJCewvYVlambgWRiuGGexFgAm6laS3rWetOcdm9eg +FoA9PKNN6xvPatbj99MPoLpBbzsI64M0yT/wTSw1pj/Nom3rwfMa2OH8Kk7c8R/r +U3xj4ZY1DTlGERvejQIDAQABMA0GCSqGSIb3DQEBCwUAA4IBAQAwyQjQ3ZgtJ3JW +r3/EPdqSUBamTfXIpOh9rXmRjPpbe+MvenqIzl4q+GnkL5mdEb1e1hdKQZgFQ5Q5 +tbcNIz6h5C07KaNtbqhZCx5c/RUEH87VeXuAuOqZHbZWJ18q0tnk+YgWER2TOkgE +RI2AslcsJBt88UUOjHX6/7J3KjPFaAjW1QV3TTsHxk14aYDYJwPdz+ijchgbOPQ0 +i+ilhzcB+qQnOC1s4xQSFo+zblTO7EgqM9KpupYfOVFh46P1Mak2W8EDvhz0livl +OROXJ6nR/13lmQdfVX6T45d+ITBwtmW2nGAh3oI3JlArGKHaW+7qnuHR72q9FSES +cEYA/wmk +-----END CERTIFICATE----- +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDWI1XPTs9jBnC5 +yXztCUvZMVm8PEPh+Vn+ncFkTcMJPkR+f3DY1PiwdcvV75H+qwMW40ZvDXLVNEzQ +MYrpavRdFPaYtBQs2AHzL8euFxJ042xhHJZ2P4uKbIQIqPCHSJKQCm9iP/sg66rp +06iHrthc91/vk4ctKGbj7yrx+7CEEv6e647g2FkMkq6lgdye4KRwXzws3vscquuB +unJWZ6CFSTAkwt4kJ7C9hWVqZuBZGK4YZ7EWACbqVpLetZ605x2b16AWgD08o03r +G89q1uP30w+gukFvOwjrgzTJP/BNLDWmP82ibevB8xrY4fwqTtzxH+tTfGPhljUN +OUYRG96NAgMBAAECggEAJ5xztBx0+nFnisZ9yG8uy6d4XPyc5gE1J4dRDdfgmyYc +j3XNjx6ePi4cHZ/qVryVnrc+AS7wrgW1q9FuS81QFKPbFdZB4SW3/p85BbgY3uxu +0Ovz3T3V9y4polx12eCP0/tKLVd+gdF2VTik9Sxfs5rC8VNN7wmJNuK4A/k15sgy +BIu/R8NlMNGQySNhtccp+dzB8uTyKx5zFZhVvnAK/3YX9BC2V4QBW9JxO4S8N0/9 +48e9Sw/fGCfQ/EFPKGCvTvfuRqJ+4t5k10FygXJ+s+y70ifYi+aSsjJBuranbLJp +g5TwhuKnTWs8Nth3YRLbcJL4VBIOehjAWy8pDMMtlQKBgQD0O8cHb8cOTGW0BijC +NDofhA2GooQUUR3WL324PXWZq0DXuBDQhJVBKWO3AYonivhhd/qWO8lea9MEmU41 +nKZ7maS4B8AJLJC08P8GL1uCIE/ezEXEi9JwC1zJiyl595Ap4lSAozH0DwjNvmGL +5mIdYg0BliqFXbloNJkNlb7INwKBgQDgdGEIWXc5Y1ncWNs6iDIV/t2MlL8vLrP0 +hpkl/QiMndOQyD6JBo0+ZqvOQTSS4NTSxBROjPxvFbEJ3eH8Pmn8gHOf46fzP1OJ +wlYv0gYzkN4FE/tN6JnO2u9pN0euyyZLM1fnEcrMWColMN8JlWjtA7Gbxm8lkfa4 +3vicaJtlWwKBgQCQYL4ZgVR0+Wit8W4qz+EEPHYafvwBXqp6sXxqa7qXawtb+q3F +9nqdGLCfwMNA+QA37ksugI1byfXmpBH902r/aiZbvAkj4zpwHH9F0r0PwbY1iSA9 +PkLahX0Gj8OnHFgWynsVyGOBWVnk9oSHxVt+7zWtGG5uhKdUGLPZugocJQKBgB61 +7bzduOFiRZ5PjhdxISE/UQL2Kz6Cbl7rt7Kp72yF/7eUnnHTMqoyFBnRdCcQmi4I +ZBrnUXbFigamlFAWHhxNWwSqeoVeychUjcRXQT/291nMhRsA02KpNA66YJV6+E9b +xBA6r/vLqGCUUkAWcFfVpIyC1xxV32MmJvAHpBN3AoGAPF3MUFiO0iKNZfst6Tm3 +rzrldLawDo98DRZ7Yb2kWlWZYqUk/Nvryvo2cns75WGSMDYVbbRp+BY7kZmNYa9K +iQzKDL54ZRu6V+getJdeAO8yXoCmnZKxt5OHvOSrQMfAmFKSwLwxBbZBfXEyuune +yfusXLtCgajpreoVIa0xWdQ= +-----END PRIVATE KEY----- diff --git a/src/allmydata/test/test_storage_client.py b/src/allmydata/test/test_storage_client.py index 524d88272..ed1dc30a9 100644 --- a/src/allmydata/test/test_storage_client.py +++ b/src/allmydata/test/test_storage_client.py @@ -65,6 +65,7 @@ from .common import ( UseTestPlugins, UseNode, SameProcessStreamEndpointAssigner, + MemoryIntroducerClient, ) from .common_web import ( do_http, @@ -505,19 +506,26 @@ class StoragePluginWebPresence(AsyncTestCase): ) +def new_tub(): + """ + Make a new ``Tub`` with a hard-coded private key. + """ + # Use a pre-generated key so the tests don't spend a lot of time + # generating new ones. + data = FilePath(__file__).sibling(b"data") + privkey = data.child(b"node.pem") + return Tub( + certData=privkey.getContent(), + ) + + def make_broker(tub_maker=None): """ Create a ``StorageFarmBroker`` with the given tub maker and an empty client configuration. """ if tub_maker is None: - # Use a pre-generated key so the tests don't spend a lot of time - # generating new ones. - data = FilePath(__file__).sibling(b"data") - privkey = data.child(b"node.pem") - tub_maker = lambda handler_overrides: Tub( - certData=privkey.getContent(), - ) + tub_maker = lambda handler_overrides: new_tub() return StorageFarmBroker(True, tub_maker, EMPTY_CLIENT_CONFIG) @@ -593,7 +601,21 @@ storage: @inlineCallbacks def test_threshold_reached(self): - introducer = Mock() + """ + ``StorageFarmBroker.when_connected_enough`` returns a ``Deferred`` which + only fires after the ``StorageFarmBroker`` has established at least as + many connections as requested. + """ + introducer = MemoryIntroducerClient( + new_tub(), + SOME_FURL, + b"", + None, + None, + None, + None, + None, + ) new_tubs = [] def make_tub(*args, **kwargs): return new_tubs.pop() @@ -601,10 +623,12 @@ storage: done = broker.when_connected_enough(5) broker.use_introducer(introducer) # subscribes to "storage" to learn of new storage nodes - subscribe = introducer.mock_calls[0] - self.assertEqual(subscribe[0], 'subscribe_to') - self.assertEqual(subscribe[1][0], 'storage') - got_announcement = subscribe[1][1] + [subscribe] = introducer.subscribed_to + self.assertEqual( + subscribe.service_name, + "storage", + ) + got_announcement = subscribe.cb data = { "service-name": "storage", From 80d2ecb815cb7b8f0ea24fff944c9fbbdcd03026 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 24 Nov 2020 14:52:17 -0500 Subject: [PATCH 05/22] news fragment --- newsfragments/3521.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3521.minor diff --git a/newsfragments/3521.minor b/newsfragments/3521.minor new file mode 100644 index 000000000..e69de29bb From 67694889da8083435a8dc5ce2ba1e16bef50c3e3 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 30 Nov 2020 17:20:39 -0500 Subject: [PATCH 06/22] Some progress towards ripping out furls-as-bytes. --- src/allmydata/storage_client.py | 5 ++--- src/allmydata/test/mutable/util.py | 2 +- src/allmydata/test/test_checker.py | 2 +- src/allmydata/test/test_client.py | 2 +- src/allmydata/test/test_introducer.py | 22 +++++++++++----------- src/allmydata/test/test_storage_client.py | 4 ++-- src/allmydata/test/test_upload.py | 2 +- 7 files changed, 19 insertions(+), 20 deletions(-) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 2b0ec5af1..0cb186c2b 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -559,6 +559,7 @@ class _FoolscapStorage(object): *nickname* is optional. """ + furl = furl.encode("utf-8") m = re.match(br'pb://(\w+)@', furl) assert m, furl tubid_s = m.group(1).lower() @@ -755,7 +756,7 @@ class NativeStorageServer(service.MultiService): else: return _FoolscapStorage.from_announcement( self._server_id, - furl.encode("utf-8"), + furl, ann, storage_server, ) @@ -767,8 +768,6 @@ class NativeStorageServer(service.MultiService): # Nope pass else: - if isinstance(furl, str): - furl = furl.encode("utf-8") # See comment above for the _storage_from_foolscap_plugin case # about passing in get_rref. storage_server = _StorageServer(get_rref=self.get_rref) diff --git a/src/allmydata/test/mutable/util.py b/src/allmydata/test/mutable/util.py index 2d14994f4..62e8d7295 100644 --- a/src/allmydata/test/mutable/util.py +++ b/src/allmydata/test/mutable/util.py @@ -239,7 +239,7 @@ def make_peer(s, i): peerid = base32.b2a(tagged_hash(b"peerid", b"%d" % i)[:20]) fss = FakeStorageServer(peerid, s) ann = { - "anonymous-storage-FURL": b"pb://%s@nowhere/fake" % (peerid,), + "anonymous-storage-FURL": "pb://%s@nowhere/fake" % (str(peerid, "utf-8"),), "permutation-seed-base32": peerid, } return Peer(peerid=peerid, storage_server=fss, announcement=ann) diff --git a/src/allmydata/test/test_checker.py b/src/allmydata/test/test_checker.py index 85b894b1f..20135688d 100644 --- a/src/allmydata/test/test_checker.py +++ b/src/allmydata/test/test_checker.py @@ -156,7 +156,7 @@ class WebResultsRendering(unittest.TestCase): for (key_s, binary_tubid, nickname) in servers: server_id = key_s tubid_b32 = base32.b2a(binary_tubid) - furl = b"pb://%s@nowhere/fake" % tubid_b32 + furl = "pb://%s@nowhere/fake" % str(tubid_b32, "utf-8") ann = { "version": 0, "service-name": "storage", "anonymous-storage-FURL": furl, diff --git a/src/allmydata/test/test_client.py b/src/allmydata/test/test_client.py index 54c5be8e5..45b9c4b14 100644 --- a/src/allmydata/test/test_client.py +++ b/src/allmydata/test/test_client.py @@ -70,7 +70,7 @@ from .matchers import ( matches_furl, ) -SOME_FURL = b"pb://abcde@nowhere/fake" +SOME_FURL = "pb://abcde@nowhere/fake" BASECONFIG = ("[client]\n" "introducer.furl = \n" diff --git a/src/allmydata/test/test_introducer.py b/src/allmydata/test/test_introducer.py index 0b2f82f62..5138318c5 100644 --- a/src/allmydata/test/test_introducer.py +++ b/src/allmydata/test/test_introducer.py @@ -213,9 +213,9 @@ class Client(AsyncTestCase): def _received(key_s, ann): announcements.append( (key_s, ann) ) ic1.subscribe_to("storage", _received) - furl1 = b"pb://62ubehyunnyhzs7r6vdonnm2hpi52w6y@127.0.0.1:36106/gydnp" - furl1a = b"pb://62ubehyunnyhzs7r6vdonnm2hpi52w6y@127.0.0.1:7777/gydnp" - furl2 = b"pb://ttwwooyunnyhzs7r6vdonnm2hpi52w6y@127.0.0.1:36106/ttwwoo" + furl1 = "pb://62ubehyunnyhzs7r6vdonnm2hpi52w6y@127.0.0.1:36106/gydnp" + furl1a = "pb://62ubehyunnyhzs7r6vdonnm2hpi52w6y@127.0.0.1:7777/gydnp" + furl2 = "pb://ttwwooyunnyhzs7r6vdonnm2hpi52w6y@127.0.0.1:36106/ttwwoo" private_key, public_key = ed25519.create_signing_keypair() public_key_str = ed25519.string_from_verifying_key(public_key) @@ -313,7 +313,7 @@ class Server(AsyncTestCase): "introducer.furl", u"my_nickname", "ver23", "oldest_version", realseq, FilePath(self.mktemp())) - furl1 = b"pb://62ubehyunnyhzs7r6vdonnm2hpi52w6y@127.0.0.1:36106/gydnp" + furl1 = "pb://62ubehyunnyhzs7r6vdonnm2hpi52w6y@127.0.0.1:36106/gydnp" private_key, _ = ed25519.create_signing_keypair() @@ -411,7 +411,7 @@ class Queue(SystemTestMixin, AsyncTestCase): c = IntroducerClient(tub2, ifurl, u"nickname", "version", "oldest", fakeseq, FilePath(self.mktemp())) - furl1 = b"pb://onug64tu@127.0.0.1:123/short" # base32("short") + furl1 = "pb://onug64tu@127.0.0.1:123/short" # base32("short") private_key, _ = ed25519.create_signing_keypair() d = introducer.disownServiceParent() @@ -753,7 +753,7 @@ class ClientInfo(AsyncTestCase): client_v2 = IntroducerClient(tub, introducer_furl, NICKNAME % u"v2", "my_version", "oldest", fakeseq, FilePath(self.mktemp())) - #furl1 = b"pb://62ubehyunnyhzs7r6vdonnm2hpi52w6y@127.0.0.1:0/swissnum" + #furl1 = "pb://62ubehyunnyhzs7r6vdonnm2hpi52w6y@127.0.0.1:0/swissnum" #ann_s = make_ann_t(client_v2, furl1, None, 10) #introducer.remote_publish_v2(ann_s, Referenceable()) subscriber = FakeRemoteReference() @@ -774,7 +774,7 @@ class Announcements(AsyncTestCase): client_v2 = IntroducerClient(tub, introducer_furl, u"nick-v2", "my_version", "oldest", fakeseq, FilePath(self.mktemp())) - furl1 = b"pb://62ubehyunnyhzs7r6vdonnm2hpi52w6y@127.0.0.1:0/swissnum" + furl1 = "pb://62ubehyunnyhzs7r6vdonnm2hpi52w6y@127.0.0.1:0/swissnum" private_key, public_key = ed25519.create_signing_keypair() public_key_str = remove_prefix(ed25519.string_from_verifying_key(public_key), b"pub-") @@ -816,7 +816,7 @@ class Announcements(AsyncTestCase): ic = c.introducer_clients[0] private_key, public_key = ed25519.create_signing_keypair() public_key_str = remove_prefix(ed25519.string_from_verifying_key(public_key), b"pub-") - furl1 = b"pb://onug64tu@127.0.0.1:123/short" # base32("short") + furl1 = "pb://onug64tu@127.0.0.1:123/short" # base32("short") ann_t = make_ann_t(ic, furl1, private_key, 1) ic.got_announcements([ann_t]) @@ -847,7 +847,7 @@ class Announcements(AsyncTestCase): # cache private_key2, public_key2 = ed25519.create_signing_keypair() public_key_str2 = remove_prefix(ed25519.string_from_verifying_key(public_key2), b"pub-") - furl3 = b"pb://onug64tu@127.0.0.1:456/short" + furl3 = "pb://onug64tu@127.0.0.1:456/short" ann_t3 = make_ann_t(ic, furl3, private_key2, 1) ic.got_announcements([ann_t3]) yield flushEventualQueue() @@ -993,7 +993,7 @@ class DecodeFurl(SyncTestCase): def test_decode(self): # make sure we have a working base64.b32decode. The one in # python2.4.[01] was broken. - furl = b'pb://t5g7egomnnktbpydbuijt6zgtmw4oqi5@127.0.0.1:51857/hfzv36i' + furl = 'pb://t5g7egomnnktbpydbuijt6zgtmw4oqi5@127.0.0.1:51857/hfzv36i' m = re.match(br'pb://(\w+)@', furl) assert m nodeid = b32decode(m.group(1).upper()) @@ -1037,7 +1037,7 @@ class Signatures(SyncTestCase): mock_tub = Mock() ic = IntroducerClient( mock_tub, - b"pb://", + "pb://", u"fake_nick", "0.0.0", "1.2.3", diff --git a/src/allmydata/test/test_storage_client.py b/src/allmydata/test/test_storage_client.py index fa3a34b15..f6bae28a7 100644 --- a/src/allmydata/test/test_storage_client.py +++ b/src/allmydata/test/test_storage_client.py @@ -88,7 +88,7 @@ from allmydata.interfaces import ( IStorageServer, ) -SOME_FURL = b"pb://abcde@nowhere/fake" +SOME_FURL = "pb://abcde@nowhere/fake" class NativeStorageServerWithVersion(NativeStorageServer): def __init__(self, version): @@ -605,7 +605,7 @@ storage: } def add_one_server(x): - data["anonymous-storage-FURL"] = b"pb://%s@nowhere/fake" % (base32.b2a(b"%d" % x),) + data["anonymous-storage-FURL"] = "pb://%s@nowhere/fake" % (base32.b2a(b"%d" % x),) tub = Mock() new_tubs.append(tub) got_announcement(b'v0-1234-%d' % x, data) diff --git a/src/allmydata/test/test_upload.py b/src/allmydata/test/test_upload.py index a6438c1fc..4817caf32 100644 --- a/src/allmydata/test/test_upload.py +++ b/src/allmydata/test/test_upload.py @@ -239,7 +239,7 @@ class FakeClient(object): node_config=EMPTY_CLIENT_CONFIG, ) for (serverid, rref) in servers: - ann = {"anonymous-storage-FURL": b"pb://%s@nowhere/fake" % base32.b2a(serverid), + ann = {"anonymous-storage-FURL": "pb://%s@nowhere/fake" % base32.b2a(serverid), "permutation-seed-base32": base32.b2a(serverid) } self.storage_broker.test_add_rref(serverid, rref, ann) self.last_servers = [s[1] for s in servers] From 22c328d6d3aff071307468c2ac12fd81e6f44f90 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 30 Nov 2020 17:41:15 -0500 Subject: [PATCH 07/22] More debytification. --- src/allmydata/introducer/common.py | 6 ++--- src/allmydata/test/test_introducer.py | 38 +++++++++++++-------------- 2 files changed, 21 insertions(+), 23 deletions(-) diff --git a/src/allmydata/introducer/common.py b/src/allmydata/introducer/common.py index 7383d507e..0ae70f505 100644 --- a/src/allmydata/introducer/common.py +++ b/src/allmydata/introducer/common.py @@ -8,14 +8,12 @@ from allmydata.util import base32, rrefutil, jsonbytes as json def get_tubid_string_from_ann(ann): furl = ann.get("anonymous-storage-FURL") or ann.get("FURL") - if isinstance(furl, unicode): - furl = furl.encode("utf-8") return get_tubid_string(furl) def get_tubid_string(furl): - m = re.match(br'pb://(\w+)@', furl) + m = re.match(r'pb://(\w+)@', furl) assert m - return m.group(1).lower() + return m.group(1).lower().encode("ascii") def sign_to_foolscap(announcement, signing_key): diff --git a/src/allmydata/test/test_introducer.py b/src/allmydata/test/test_introducer.py index 5138318c5..cb3df7998 100644 --- a/src/allmydata/test/test_introducer.py +++ b/src/allmydata/test/test_introducer.py @@ -239,7 +239,7 @@ class Client(AsyncTestCase): self.failUnlessEqual(len(announcements), 1) key_s,ann = announcements[0] self.failUnlessEqual(key_s, pubkey_s) - self.failUnlessEqual(ensure_binary(ann["anonymous-storage-FURL"]), furl1) + self.failUnlessEqual(ann["anonymous-storage-FURL"], furl1) self.failUnlessEqual(ann["my-version"], "ver23") d.addCallback(_then1) @@ -273,7 +273,7 @@ class Client(AsyncTestCase): self.failUnlessEqual(len(announcements), 2) key_s,ann = announcements[-1] self.failUnlessEqual(key_s, pubkey_s) - self.failUnlessEqual(ensure_binary(ann["anonymous-storage-FURL"]), furl1) + self.failUnlessEqual(ann["anonymous-storage-FURL"], furl1) self.failUnlessEqual(ann["my-version"], "ver24") d.addCallback(_then3) @@ -285,7 +285,7 @@ class Client(AsyncTestCase): self.failUnlessEqual(len(announcements), 3) key_s,ann = announcements[-1] self.failUnlessEqual(key_s, pubkey_s) - self.failUnlessEqual(ensure_binary(ann["anonymous-storage-FURL"]), furl1a) + self.failUnlessEqual(ann["anonymous-storage-FURL"], furl1a) self.failUnlessEqual(ann["my-version"], "ver23") d.addCallback(_then4) @@ -301,7 +301,7 @@ class Client(AsyncTestCase): self.failUnlessEqual(len(announcements2), 1) key_s,ann = announcements2[-1] self.failUnlessEqual(key_s, pubkey_s) - self.failUnlessEqual(ensure_binary(ann["anonymous-storage-FURL"]), furl1a) + self.failUnlessEqual(ann["anonymous-storage-FURL"], furl1a) self.failUnlessEqual(ann["my-version"], "ver23") d.addCallback(_then5) return d @@ -433,7 +433,7 @@ class Queue(SystemTestMixin, AsyncTestCase): def _done(ign): v = introducer.get_announcements()[0] furl = v.announcement["anonymous-storage-FURL"] - self.failUnlessEqual(ensure_binary(furl), furl1) + self.failUnlessEqual(furl, furl1) d.addCallback(_done) # now let the ack get back @@ -459,7 +459,7 @@ class SystemTest(SystemTestMixin, AsyncTestCase): iff = os.path.join(self.basedir, "introducer.furl") tub = self.central_tub ifurl = self.central_tub.registerReference(introducer, furlFile=iff) - self.introducer_furl = ifurl.encode("utf-8") + self.introducer_furl = ifurl # we have 5 clients who publish themselves as storage servers, and a # sixth which does which not. All 6 clients subscriber to hear about @@ -500,7 +500,7 @@ class SystemTest(SystemTestMixin, AsyncTestCase): subscribing_clients.append(c) expected_announcements[i] += 1 # all expect a 'storage' announcement - node_furl = tub.registerReference(Referenceable()).encode("utf-8") + node_furl = tub.registerReference(Referenceable()) private_key, public_key = ed25519.create_signing_keypair() public_key_str = ed25519.string_from_verifying_key(public_key) privkeys[i] = private_key @@ -517,7 +517,7 @@ class SystemTest(SystemTestMixin, AsyncTestCase): if i == 2: # also publish something that nobody cares about - boring_furl = tub.registerReference(Referenceable()).encode("utf-8") + boring_furl = tub.registerReference(Referenceable()) c.publish("boring", make_ann(boring_furl), private_key) c.setServiceParent(self.parent) @@ -655,7 +655,7 @@ class SystemTest(SystemTestMixin, AsyncTestCase): self.create_tub(self.central_portnum) newfurl = self.central_tub.registerReference(self.the_introducer, furlFile=iff) - assert ensure_binary(newfurl) == self.introducer_furl + assert newfurl == self.introducer_furl d.addCallback(_restart_introducer_tub) d.addCallback(_wait_for_connected) @@ -707,7 +707,7 @@ class SystemTest(SystemTestMixin, AsyncTestCase): self.the_introducer = introducer newfurl = self.central_tub.registerReference(self.the_introducer, furlFile=iff) - assert ensure_binary(newfurl) == self.introducer_furl + assert newfurl == self.introducer_furl d.addCallback(_restart_introducer) d.addCallback(_wait_for_connected) @@ -789,7 +789,7 @@ class Announcements(AsyncTestCase): self.failUnlessEqual(a[0].nickname, u"nick-v2") self.failUnlessEqual(a[0].service_name, "storage") self.failUnlessEqual(a[0].version, "my_version") - self.failUnlessEqual(ensure_binary(a[0].announcement["anonymous-storage-FURL"]), furl1) + self.failUnlessEqual(a[0].announcement["anonymous-storage-FURL"], furl1) def _load_cache(self, cache_filepath): with cache_filepath.open() as f: @@ -827,12 +827,12 @@ class Announcements(AsyncTestCase): self.failUnlessEqual(len(announcements), 1) self.failUnlessEqual(ensure_binary(announcements[0]['key_s']), public_key_str) ann = announcements[0]["ann"] - self.failUnlessEqual(ensure_binary(ann["anonymous-storage-FURL"]), furl1) + self.failUnlessEqual(ann["anonymous-storage-FURL"], furl1) self.failUnlessEqual(ann["seqnum"], 1) # a new announcement that replaces the first should replace the # cached entry, not duplicate it - furl2 = furl1 + b"er" + furl2 = furl1 + "er" ann_t2 = make_ann_t(ic, furl2, private_key, 2) ic.got_announcements([ann_t2]) yield flushEventualQueue() @@ -840,7 +840,7 @@ class Announcements(AsyncTestCase): self.failUnlessEqual(len(announcements), 1) self.failUnlessEqual(ensure_binary(announcements[0]['key_s']), public_key_str) ann = announcements[0]["ann"] - self.failUnlessEqual(ensure_binary(ann["anonymous-storage-FURL"]), furl2) + self.failUnlessEqual(ann["anonymous-storage-FURL"], furl2) self.failUnlessEqual(ann["seqnum"], 2) # but a third announcement with a different key should add to the @@ -857,7 +857,7 @@ class Announcements(AsyncTestCase): self.failUnlessEqual(set([public_key_str, public_key_str2]), set([ensure_binary(a["key_s"]) for a in announcements])) self.failUnlessEqual(set([furl2, furl3]), - set([ensure_binary(a["ann"]["anonymous-storage-FURL"]) + set([a["ann"]["anonymous-storage-FURL"] for a in announcements])) # test loading @@ -873,9 +873,9 @@ class Announcements(AsyncTestCase): yield flushEventualQueue() self.failUnless(public_key_str in announcements) - self.failUnlessEqual(ensure_binary(announcements[public_key_str]["anonymous-storage-FURL"]), + self.failUnlessEqual(announcements[public_key_str]["anonymous-storage-FURL"], furl2) - self.failUnlessEqual(ensure_binary(announcements[public_key_str2]["anonymous-storage-FURL"]), + self.failUnlessEqual(announcements[public_key_str2]["anonymous-storage-FURL"], furl3) c2 = yield create_client(basedir) @@ -994,9 +994,9 @@ class DecodeFurl(SyncTestCase): # make sure we have a working base64.b32decode. The one in # python2.4.[01] was broken. furl = 'pb://t5g7egomnnktbpydbuijt6zgtmw4oqi5@127.0.0.1:51857/hfzv36i' - m = re.match(br'pb://(\w+)@', furl) + m = re.match(r'pb://(\w+)@', furl) assert m - nodeid = b32decode(m.group(1).upper()) + nodeid = b32decode(m.group(1).upper().encode("ascii")) self.failUnlessEqual(nodeid, b"\x9fM\xf2\x19\xcckU0\xbf\x03\r\x10\x99\xfb&\x9b-\xc7A\x1d") class Signatures(SyncTestCase): From 17ec22ab84303863556beb9535af60a44c8e825d Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 30 Nov 2020 17:55:29 -0500 Subject: [PATCH 08/22] Even more debytification of furls. --- src/allmydata/test/test_storage_client.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/allmydata/test/test_storage_client.py b/src/allmydata/test/test_storage_client.py index f6bae28a7..1f5c1306d 100644 --- a/src/allmydata/test/test_storage_client.py +++ b/src/allmydata/test/test_storage_client.py @@ -295,7 +295,7 @@ class PluginMatchedAnnouncement(SyncTestCase): # notice how the announcement is for a different storage plugin # than the one that is enabled. u"name": u"tahoe-lafs-dummy-v2", - u"storage-server-FURL": SOME_FURL.decode("ascii"), + u"storage-server-FURL": SOME_FURL, }], } self.publish(server_id, ann, self.introducer_client) @@ -323,7 +323,7 @@ class PluginMatchedAnnouncement(SyncTestCase): u"storage-options": [{ # and this announcement is for a plugin with a matching name u"name": plugin_name, - u"storage-server-FURL": SOME_FURL.decode("ascii"), + u"storage-server-FURL": SOME_FURL, }], } self.publish(server_id, ann, self.introducer_client) @@ -374,7 +374,7 @@ class PluginMatchedAnnouncement(SyncTestCase): u"storage-options": [{ # and this announcement is for a plugin with a matching name u"name": plugin_name, - u"storage-server-FURL": SOME_FURL.decode("ascii"), + u"storage-server-FURL": SOME_FURL, }], } self.publish(server_id, ann, self.introducer_client) @@ -525,7 +525,7 @@ storage: ann: anonymous-storage-FURL: {furl} permutation-seed-base32: aaaaaaaaaaaaaaaaaaaaaaaa -""".format(furl=SOME_FURL.decode("utf-8")) +""".format(furl=SOME_FURL) servers = yamlutil.safe_load(servers_yaml) permseed = base32.a2b(b"aaaaaaaaaaaaaaaaaaaaaaaa") broker.set_static_servers(servers["storage"]) @@ -541,7 +541,7 @@ storage: ann2 = { "service-name": "storage", - "anonymous-storage-FURL": "pb://{}@nowhere/fake2".format(base32.b2a(b"1")), + "anonymous-storage-FURL": "pb://{}@nowhere/fake2".format(str(base32.b2a(b"1"), "utf-8")), "permutation-seed-base32": "bbbbbbbbbbbbbbbbbbbbbbbb", } broker._got_announcement(key_s, ann2) @@ -605,7 +605,7 @@ storage: } def add_one_server(x): - data["anonymous-storage-FURL"] = "pb://%s@nowhere/fake" % (base32.b2a(b"%d" % x),) + data["anonymous-storage-FURL"] = "pb://%s@nowhere/fake" % (str(base32.b2a(b"%d" % x), "ascii"),) tub = Mock() new_tubs.append(tub) got_announcement(b'v0-1234-%d' % x, data) From 8a47227e9bc685adc837fa982375fd7d7f24fbf4 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 30 Nov 2020 17:56:46 -0500 Subject: [PATCH 09/22] News file. --- newsfragments/3544.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3544.minor diff --git a/newsfragments/3544.minor b/newsfragments/3544.minor new file mode 100644 index 000000000..e69de29bb From d5ba1be5bcc9a0f41765e3398412a6b6572aef9b Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 30 Nov 2020 17:56:50 -0500 Subject: [PATCH 10/22] Passing tests. --- src/allmydata/test/test_upload.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/test_upload.py b/src/allmydata/test/test_upload.py index 4817caf32..94d7575c3 100644 --- a/src/allmydata/test/test_upload.py +++ b/src/allmydata/test/test_upload.py @@ -239,7 +239,7 @@ class FakeClient(object): node_config=EMPTY_CLIENT_CONFIG, ) for (serverid, rref) in servers: - ann = {"anonymous-storage-FURL": "pb://%s@nowhere/fake" % base32.b2a(serverid), + ann = {"anonymous-storage-FURL": "pb://%s@nowhere/fake" % str(base32.b2a(serverid), "ascii"), "permutation-seed-base32": base32.b2a(serverid) } self.storage_broker.test_add_rref(serverid, rref, ann) self.last_servers = [s[1] for s in servers] From 0386924d009e28fb7d7bf46ad2e876708b19ab38 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 2 Dec 2020 15:26:25 -0500 Subject: [PATCH 11/22] Fix flake. --- src/allmydata/introducer/common.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/allmydata/introducer/common.py b/src/allmydata/introducer/common.py index 0ae70f505..8ccb3c2e5 100644 --- a/src/allmydata/introducer/common.py +++ b/src/allmydata/introducer/common.py @@ -1,5 +1,3 @@ -from past.builtins import unicode - import re from allmydata.crypto.util import remove_prefix from allmydata.crypto import ed25519 From 15735e70bccd98bd69e47c1d14d4c90d4da1fc99 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 4 Dec 2020 11:18:51 -0500 Subject: [PATCH 12/22] Port to Python 3. --- src/allmydata/introducer/client.py | 30 +++++++++++++++++++++--------- src/allmydata/introducer/common.py | 12 ++++++++++++ src/allmydata/util/_python3.py | 2 ++ 3 files changed, 35 insertions(+), 9 deletions(-) diff --git a/src/allmydata/introducer/client.py b/src/allmydata/introducer/client.py index f54595221..fa1e1efe8 100644 --- a/src/allmydata/introducer/client.py +++ b/src/allmydata/introducer/client.py @@ -1,4 +1,16 @@ -from past.builtins import unicode, long +""" +Ported to Python 3. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 +from past.builtins import long + from six import ensure_text import time @@ -27,11 +39,11 @@ class IntroducerClient(service.Service, Referenceable): nickname, my_version, oldest_supported, sequencer, cache_filepath): self._tub = tub - if isinstance(introducer_furl, unicode): + if isinstance(introducer_furl, str): introducer_furl = introducer_furl.encode("utf-8") self.introducer_furl = introducer_furl - assert type(nickname) is unicode + assert isinstance(nickname, str) self._nickname = nickname self._my_version = my_version self._oldest_supported = oldest_supported @@ -114,7 +126,7 @@ class IntroducerClient(service.Service, Referenceable): def _save_announcements(self): announcements = [] - for _, value in self._inbound_announcements.items(): + for value in self._inbound_announcements.values(): ann, key_s, time_stamp = value # On Python 2, bytes strings are encoded into YAML Unicode strings. # On Python 3, bytes are encoded as YAML bytes. To minimize @@ -125,7 +137,7 @@ class IntroducerClient(service.Service, Referenceable): } announcements.append(server_params) announcement_cache_yaml = yamlutil.safe_dump(announcements) - if isinstance(announcement_cache_yaml, unicode): + if isinstance(announcement_cache_yaml, str): announcement_cache_yaml = announcement_cache_yaml.encode("utf-8") self._cache_filepath.setContent(announcement_cache_yaml) @@ -170,7 +182,7 @@ class IntroducerClient(service.Service, Referenceable): self._local_subscribers.append( (service_name,cb,args,kwargs) ) self._subscribed_service_names.add(service_name) self._maybe_subscribe() - for index,(ann,key_s,when) in self._inbound_announcements.items(): + for index,(ann,key_s,when) in list(self._inbound_announcements.items()): precondition(isinstance(key_s, bytes), key_s) servicename = index[0] if servicename == service_name: @@ -215,7 +227,7 @@ class IntroducerClient(service.Service, Referenceable): self._outbound_announcements[service_name] = ann_d # publish all announcements with the new seqnum and nonce - for service_name,ann_d in self._outbound_announcements.items(): + for service_name,ann_d in list(self._outbound_announcements.items()): ann_d["seqnum"] = current_seqnum ann_d["nonce"] = current_nonce ann_t = sign_to_foolscap(ann_d, signing_key) @@ -227,7 +239,7 @@ class IntroducerClient(service.Service, Referenceable): self.log("want to publish, but no introducer yet", level=log.NOISY) return # this re-publishes everything. The Introducer ignores duplicates - for ann_t in self._published_announcements.values(): + for ann_t in list(self._published_announcements.values()): self._debug_counts["outbound_message"] += 1 self._debug_outstanding += 1 d = self._publisher.callRemote("publish_v2", ann_t, self._canary) @@ -267,7 +279,7 @@ class IntroducerClient(service.Service, Referenceable): return # for ASCII values, simplejson might give us unicode *or* bytes if "nickname" in ann and isinstance(ann["nickname"], bytes): - ann["nickname"] = unicode(ann["nickname"]) + ann["nickname"] = str(ann["nickname"]) nick_s = ann.get("nickname",u"").encode("utf-8") lp2 = self.log(format="announcement for nickname '%(nick)s', service=%(svc)s: %(ann)s", nick=nick_s, svc=service_name, ann=ann, umid="BoKEag") diff --git a/src/allmydata/introducer/common.py b/src/allmydata/introducer/common.py index 8ccb3c2e5..f67aad203 100644 --- a/src/allmydata/introducer/common.py +++ b/src/allmydata/introducer/common.py @@ -1,3 +1,15 @@ +""" +Ported to Python 3. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + import re from allmydata.crypto.util import remove_prefix from allmydata.crypto import ed25519 diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 93e488c6e..2f02fdaa1 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -51,6 +51,8 @@ PORTED_MODULES = [ "allmydata.immutable.offloaded", "allmydata.immutable.upload", "allmydata.interfaces", + "allmydata.introducer.client", + "allmydata.introducer.common", "allmydata.introducer.interfaces", "allmydata.monitor", "allmydata.mutable.checker", From 22dcd726e67c8c37882e529e5f0a66d735e26d9b Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 4 Dec 2020 11:36:17 -0500 Subject: [PATCH 13/22] Port to Python 3. --- src/allmydata/introducer/server.py | 31 +++++++++++++++++++++--------- src/allmydata/util/_python3.py | 1 + 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/src/allmydata/introducer/server.py b/src/allmydata/introducer/server.py index e41bff14b..870b8aba1 100644 --- a/src/allmydata/introducer/server.py +++ b/src/allmydata/introducer/server.py @@ -1,3 +1,16 @@ +""" +Ported to Python 3. +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from past.builtins import long from six import ensure_str, ensure_text @@ -157,7 +170,7 @@ class IntroducerService(service.MultiService, Referenceable): # 'subscriber_info' is a dict, provided directly by v2 clients. The # expected keys are: version, nickname, app-versions, my-version, # oldest-supported - self._subscribers = {} + self._subscribers = dictutil.UnicodeKeyDict({}) self._debug_counts = {"inbound_message": 0, "inbound_duplicate": 0, @@ -181,7 +194,7 @@ class IntroducerService(service.MultiService, Referenceable): def get_announcements(self): """Return a list of AnnouncementDescriptor for all announcements""" announcements = [] - for (index, (_, canary, ann, when)) in self._announcements.items(): + for (index, (_, canary, ann, when)) in list(self._announcements.items()): ad = AnnouncementDescriptor(when, index, canary, ann) announcements.append(ad) return announcements @@ -189,8 +202,8 @@ class IntroducerService(service.MultiService, Referenceable): def get_subscribers(self): """Return a list of SubscriberDescriptor objects for all subscribers""" s = [] - for service_name, subscriptions in self._subscribers.items(): - for rref,(subscriber_info,when) in subscriptions.items(): + for service_name, subscriptions in list(self._subscribers.items()): + for rref,(subscriber_info,when) in list(subscriptions.items()): # note that if the subscriber didn't do Tub.setLocation, # tubid will be None. Also, subscribers do not tell us which # pubkey they use; only publishers do that. @@ -281,7 +294,7 @@ class IntroducerService(service.MultiService, Referenceable): def remote_subscribe_v2(self, subscriber, service_name, subscriber_info): self.log("introducer: subscription[%s] request at %s" % (service_name, subscriber), umid="U3uzLg") - service_name = ensure_str(service_name) + service_name = ensure_text(service_name) subscriber_info = dictutil.UnicodeKeyDict({ ensure_text(k): v for (k, v) in subscriber_info.items() }) @@ -307,11 +320,11 @@ class IntroducerService(service.MultiService, Referenceable): subscribers.pop(subscriber, None) subscriber.notifyOnDisconnect(_remove) + # Make sure types are correct: + for k in self._announcements: + assert isinstance(k[0], type(service_name)) + # now tell them about any announcements they're interested in - assert {type(service_name)}.issuperset( - set(type(k[0]) for k in self._announcements)), ( - service_name, self._announcements.keys() - ) announcements = set( [ ann_t for idx,(ann_t,canary,ann,when) in self._announcements.items() diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 2f02fdaa1..9763c35d7 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -54,6 +54,7 @@ PORTED_MODULES = [ "allmydata.introducer.client", "allmydata.introducer.common", "allmydata.introducer.interfaces", + "allmydata.introducer.server", "allmydata.monitor", "allmydata.mutable.checker", "allmydata.mutable.common", From afc443bff056cdd9111c48a23cf66802d1d64847 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 4 Dec 2020 11:37:08 -0500 Subject: [PATCH 14/22] News file. --- newsfragments/3546.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3546.minor diff --git a/newsfragments/3546.minor b/newsfragments/3546.minor new file mode 100644 index 000000000..e69de29bb From c1a699dede317cd6ef60613f97e1aa90b1355d4f Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 4 Dec 2020 11:39:57 -0500 Subject: [PATCH 15/22] Flake. --- src/allmydata/introducer/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/introducer/server.py b/src/allmydata/introducer/server.py index 870b8aba1..cd3d4a68a 100644 --- a/src/allmydata/introducer/server.py +++ b/src/allmydata/introducer/server.py @@ -12,7 +12,7 @@ from future.utils import PY2 if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from past.builtins import long -from six import ensure_str, ensure_text +from six import ensure_text import time, os.path, textwrap from zope.interface import implementer From 386f30fa797b98de7fccf6a5c06960936849d27e Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Sun, 6 Dec 2020 10:30:08 -0500 Subject: [PATCH 16/22] Remove Mock from test_threshold_reached --- src/allmydata/test/no_network.py | 3 + src/allmydata/test/test_storage_client.py | 88 ++++++++++++++++++++--- 2 files changed, 82 insertions(+), 9 deletions(-) diff --git a/src/allmydata/test/no_network.py b/src/allmydata/test/no_network.py index 54095f15d..f5f425dbc 100644 --- a/src/allmydata/test/no_network.py +++ b/src/allmydata/test/no_network.py @@ -89,6 +89,9 @@ class LocalWrapper(object): self.counter_by_methname = {} self._fireEventually = fireEventually + def getLocationHints(self): + return [] + def _clear_counters(self): self.counter_by_methname = {} diff --git a/src/allmydata/test/test_storage_client.py b/src/allmydata/test/test_storage_client.py index ed1dc30a9..db0b1467f 100644 --- a/src/allmydata/test/test_storage_client.py +++ b/src/allmydata/test/test_storage_client.py @@ -17,7 +17,6 @@ from json import ( ) import hashlib -from mock import Mock from fixtures import ( TempDir, ) @@ -44,12 +43,20 @@ from hyperlink import ( URL, ) +import attr + +from twisted.internet.interfaces import ( + IStreamClientEndpoint, +) from twisted.application.service import ( Service, ) from twisted.trial import unittest -from twisted.internet.defer import succeed, inlineCallbacks +from twisted.internet.defer import ( + Deferred, + inlineCallbacks, +) from twisted.python.filepath import ( FilePath, ) @@ -57,7 +64,11 @@ from twisted.python.filepath import ( from foolscap.api import ( Tub, ) +from foolscap.ipb import ( + IConnectionHintHandler, +) +from .no_network import LocalWrapper from .common import ( EMPTY_CLIENT_CONFIG, SyncTestCase, @@ -84,6 +95,9 @@ from allmydata.storage_client import ( _FoolscapStorage, _NullStorage, ) +from ..storage.server import ( + StorageServer, +) from allmydata.interfaces import ( IConnectionStatus, IStorageServer, @@ -529,6 +543,47 @@ def make_broker(tub_maker=None): return StorageFarmBroker(True, tub_maker, EMPTY_CLIENT_CONFIG) +@implementer(IStreamClientEndpoint) +@attr.s +class SpyEndpoint(object): + """ + Observe and record connection attempts. + + :ivar list _append: A callable that accepts two-tuples. For each + attempted connection, it will be called with ``Deferred`` that was + returned and the ``Factory`` that was passed in. + """ + _append = attr.ib() + + def connect(self, factory): + """ + Record the connection attempt. + + :return: A ``Deferred`` that ``SpyEndpoint`` will not fire. + """ + d = Deferred() + self._append((d, factory)) + return d + + +@implementer(IConnectionHintHandler) +@attr.s +class SpyHandler(object): + """ + A Foolscap connection hint handler for the "spy" hint type. Connections + are handled by just observing and recording them. + + :ivar list _connects: A list containing one element for each connection + attempted with this handler. Each element is a two-tuple of the + ``Deferred`` that was returned from ``connect`` and the factory that + was passed to ``connect``. + """ + _connects = attr.ib(default=attr.Factory(list)) + + def hint_to_endpoint(self, hint, reactor, update_status): + return (SpyEndpoint(self._connects.append), hint) + + class TestStorageFarmBroker(unittest.TestCase): def test_static_servers(self): @@ -620,6 +675,11 @@ storage: def make_tub(*args, **kwargs): return new_tubs.pop() broker = make_broker(make_tub) + # Start the broker so that it will start Tubs attached to it so they + # will attempt to make connections as necessary so that we can observe + # those connections. + broker.startService() + self.addCleanup(broker.stopService) done = broker.when_connected_enough(5) broker.use_introducer(introducer) # subscribes to "storage" to learn of new storage nodes @@ -637,15 +697,25 @@ storage: } def add_one_server(x): - data["anonymous-storage-FURL"] = b"pb://%s@nowhere/fake" % (base32.b2a(b"%d" % x),) - tub = Mock() + data["anonymous-storage-FURL"] = b"pb://%s@spy:nowhere/fake" % (base32.b2a(b"%d" % x),) + tub = Tub() + connects = [] + spy = SpyHandler(connects) + tub.addConnectionHintHandler("spy", spy) new_tubs.append(tub) got_announcement(b'v0-1234-%d' % x, data) - self.assertEqual(tub.mock_calls[-1][0], 'connectTo') - got_connection = tub.mock_calls[-1][1][1] - rref = Mock() - rref.callRemote = Mock(return_value=succeed(1234)) - got_connection(rref) + + self.assertEqual( + 1, len(connects), + "Expected one connection attempt, got {!r} instead".format(connects), + ) + + # Skip over all the Foolscap negotiation. It's complex with lots + # of pieces and I don't want to figure out how to fake + # it. -exarkun + native = broker.servers[b"v0-1234-%d" % (x,)] + rref = LocalWrapper(StorageServer(self.mktemp(), b"x" * 20)) + native._got_connection(rref) # first 4 shouldn't trigger connected_threashold for x in range(4): From 8278fa9c620320b0627eb3a98910d68d1ac24aa3 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Sun, 6 Dec 2020 10:39:25 -0500 Subject: [PATCH 17/22] Name the pem so it gets installed Also use the new_tub helper that reads the pem so we don't have to generate a new cert in this test --- src/allmydata/test/data/{node.pem => node.pem.txt} | 0 src/allmydata/test/test_storage_client.py | 4 ++-- 2 files changed, 2 insertions(+), 2 deletions(-) rename src/allmydata/test/data/{node.pem => node.pem.txt} (100%) diff --git a/src/allmydata/test/data/node.pem b/src/allmydata/test/data/node.pem.txt similarity index 100% rename from src/allmydata/test/data/node.pem rename to src/allmydata/test/data/node.pem.txt diff --git a/src/allmydata/test/test_storage_client.py b/src/allmydata/test/test_storage_client.py index db0b1467f..6b89c5bf0 100644 --- a/src/allmydata/test/test_storage_client.py +++ b/src/allmydata/test/test_storage_client.py @@ -527,7 +527,7 @@ def new_tub(): # Use a pre-generated key so the tests don't spend a lot of time # generating new ones. data = FilePath(__file__).sibling(b"data") - privkey = data.child(b"node.pem") + privkey = data.child(b"node.pem.txt") return Tub( certData=privkey.getContent(), ) @@ -698,7 +698,7 @@ storage: def add_one_server(x): data["anonymous-storage-FURL"] = b"pb://%s@spy:nowhere/fake" % (base32.b2a(b"%d" % x),) - tub = Tub() + tub = new_tub() connects = [] spy = SpyHandler(connects) tub.addConnectionHintHandler("spy", spy) From d83e4790c335bd0231021e04433abbf626961722 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 7 Dec 2020 13:14:00 -0500 Subject: [PATCH 18/22] Let subscribed_to get initialized right --- src/allmydata/test/test_storage_client.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/allmydata/test/test_storage_client.py b/src/allmydata/test/test_storage_client.py index 531193804..913d9bf9c 100644 --- a/src/allmydata/test/test_storage_client.py +++ b/src/allmydata/test/test_storage_client.py @@ -669,7 +669,6 @@ storage: None, None, None, - None, ) new_tubs = [] def make_tub(*args, **kwargs): From 0b7c0bf451d4ab745b64651c78838f143ee58115 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 8 Dec 2020 10:28:47 -0500 Subject: [PATCH 19/22] Document furl type. --- src/allmydata/storage_client.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 0cb186c2b..cef9f9d93 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -558,6 +558,9 @@ class _FoolscapStorage(object): } *nickname* is optional. + + The furl will be a Unicode string on Python 3; on Python 2 it will be + either a native (bytes) string or a Unicode string. """ furl = furl.encode("utf-8") m = re.match(br'pb://(\w+)@', furl) From eddf7fd8f906b5903ad4c5cab67031ff78937489 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 8 Dec 2020 10:34:25 -0500 Subject: [PATCH 20/22] Remove the pre-generated key/certificate --- src/allmydata/test/data/node.pem.txt | 45 ----------------------- src/allmydata/test/test_storage_client.py | 12 +++--- 2 files changed, 5 insertions(+), 52 deletions(-) delete mode 100644 src/allmydata/test/data/node.pem.txt diff --git a/src/allmydata/test/data/node.pem.txt b/src/allmydata/test/data/node.pem.txt deleted file mode 100644 index dd0928cdd..000000000 --- a/src/allmydata/test/data/node.pem.txt +++ /dev/null @@ -1,45 +0,0 @@ ------BEGIN CERTIFICATE----- -MIICojCCAYoCAQEwDQYJKoZIhvcNAQELBQAwFzEVMBMGA1UEAwwMbmV3cGJfdGhp -bmd5MB4XDTIwMDEwMjAxNDAzM1oXDTIxMDEwMTAxNDAzM1owFzEVMBMGA1UEAwwM -bmV3cGJfdGhpbmd5MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA1iNV -z07PYwZwucl87QlL2TFZvDxD4flZ/p3BZE3DCT5Efn9w2NT4sHXL1e+R/qsDFuNG -bw1y1TRM0DGK6Wr0XRT2mLQULNgB8y/HrhcSdONsYRyWdj+LimyECKjwh0iSkApv -Yj/7IOuq6dOoh67YXPdf75OHLShm4+8q8fuwhBL+nuuO4NhZDJKupYHcnuCkcF88 -LN77HKrrgbpyVmeghUkwJMLeJCewvYVlambgWRiuGGexFgAm6laS3rWetOcdm9eg -FoA9PKNN6xvPatbj99MPoLpBbzsI64M0yT/wTSw1pj/Nom3rwfMa2OH8Kk7c8R/r -U3xj4ZY1DTlGERvejQIDAQABMA0GCSqGSIb3DQEBCwUAA4IBAQAwyQjQ3ZgtJ3JW -r3/EPdqSUBamTfXIpOh9rXmRjPpbe+MvenqIzl4q+GnkL5mdEb1e1hdKQZgFQ5Q5 -tbcNIz6h5C07KaNtbqhZCx5c/RUEH87VeXuAuOqZHbZWJ18q0tnk+YgWER2TOkgE -RI2AslcsJBt88UUOjHX6/7J3KjPFaAjW1QV3TTsHxk14aYDYJwPdz+ijchgbOPQ0 -i+ilhzcB+qQnOC1s4xQSFo+zblTO7EgqM9KpupYfOVFh46P1Mak2W8EDvhz0livl -OROXJ6nR/13lmQdfVX6T45d+ITBwtmW2nGAh3oI3JlArGKHaW+7qnuHR72q9FSES -cEYA/wmk ------END CERTIFICATE----- ------BEGIN PRIVATE KEY----- -MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDWI1XPTs9jBnC5 -yXztCUvZMVm8PEPh+Vn+ncFkTcMJPkR+f3DY1PiwdcvV75H+qwMW40ZvDXLVNEzQ -MYrpavRdFPaYtBQs2AHzL8euFxJ042xhHJZ2P4uKbIQIqPCHSJKQCm9iP/sg66rp -06iHrthc91/vk4ctKGbj7yrx+7CEEv6e647g2FkMkq6lgdye4KRwXzws3vscquuB -unJWZ6CFSTAkwt4kJ7C9hWVqZuBZGK4YZ7EWACbqVpLetZ605x2b16AWgD08o03r -G89q1uP30w+gukFvOwjrgzTJP/BNLDWmP82ibevB8xrY4fwqTtzxH+tTfGPhljUN -OUYRG96NAgMBAAECggEAJ5xztBx0+nFnisZ9yG8uy6d4XPyc5gE1J4dRDdfgmyYc -j3XNjx6ePi4cHZ/qVryVnrc+AS7wrgW1q9FuS81QFKPbFdZB4SW3/p85BbgY3uxu -0Ovz3T3V9y4polx12eCP0/tKLVd+gdF2VTik9Sxfs5rC8VNN7wmJNuK4A/k15sgy -BIu/R8NlMNGQySNhtccp+dzB8uTyKx5zFZhVvnAK/3YX9BC2V4QBW9JxO4S8N0/9 -48e9Sw/fGCfQ/EFPKGCvTvfuRqJ+4t5k10FygXJ+s+y70ifYi+aSsjJBuranbLJp -g5TwhuKnTWs8Nth3YRLbcJL4VBIOehjAWy8pDMMtlQKBgQD0O8cHb8cOTGW0BijC -NDofhA2GooQUUR3WL324PXWZq0DXuBDQhJVBKWO3AYonivhhd/qWO8lea9MEmU41 -nKZ7maS4B8AJLJC08P8GL1uCIE/ezEXEi9JwC1zJiyl595Ap4lSAozH0DwjNvmGL -5mIdYg0BliqFXbloNJkNlb7INwKBgQDgdGEIWXc5Y1ncWNs6iDIV/t2MlL8vLrP0 -hpkl/QiMndOQyD6JBo0+ZqvOQTSS4NTSxBROjPxvFbEJ3eH8Pmn8gHOf46fzP1OJ -wlYv0gYzkN4FE/tN6JnO2u9pN0euyyZLM1fnEcrMWColMN8JlWjtA7Gbxm8lkfa4 -3vicaJtlWwKBgQCQYL4ZgVR0+Wit8W4qz+EEPHYafvwBXqp6sXxqa7qXawtb+q3F -9nqdGLCfwMNA+QA37ksugI1byfXmpBH902r/aiZbvAkj4zpwHH9F0r0PwbY1iSA9 -PkLahX0Gj8OnHFgWynsVyGOBWVnk9oSHxVt+7zWtGG5uhKdUGLPZugocJQKBgB61 -7bzduOFiRZ5PjhdxISE/UQL2Kz6Cbl7rt7Kp72yF/7eUnnHTMqoyFBnRdCcQmi4I -ZBrnUXbFigamlFAWHhxNWwSqeoVeychUjcRXQT/291nMhRsA02KpNA66YJV6+E9b -xBA6r/vLqGCUUkAWcFfVpIyC1xxV32MmJvAHpBN3AoGAPF3MUFiO0iKNZfst6Tm3 -rzrldLawDo98DRZ7Yb2kWlWZYqUk/Nvryvo2cns75WGSMDYVbbRp+BY7kZmNYa9K -iQzKDL54ZRu6V+getJdeAO8yXoCmnZKxt5OHvOSrQMfAmFKSwLwxBbZBfXEyuune -yfusXLtCgajpreoVIa0xWdQ= ------END PRIVATE KEY----- diff --git a/src/allmydata/test/test_storage_client.py b/src/allmydata/test/test_storage_client.py index 913d9bf9c..44e481d52 100644 --- a/src/allmydata/test/test_storage_client.py +++ b/src/allmydata/test/test_storage_client.py @@ -520,17 +520,15 @@ class StoragePluginWebPresence(AsyncTestCase): ) +_aCertPEM = Tub().myCertificate.dumpPEM() def new_tub(): """ Make a new ``Tub`` with a hard-coded private key. """ - # Use a pre-generated key so the tests don't spend a lot of time - # generating new ones. - data = FilePath(__file__).sibling(b"data") - privkey = data.child(b"node.pem.txt") - return Tub( - certData=privkey.getContent(), - ) + # Use a private key / certificate generated by Tub how it wants. But just + # re-use the same one every time so we don't waste a lot of time + # generating them over and over in the tests. + return Tub(certData=_aCertPEM) def make_broker(tub_maker=None): From 69d3dad64691041161e0132925ccac89a939f21d Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 8 Dec 2020 10:37:31 -0500 Subject: [PATCH 21/22] Get rid of remote_host / get_remote_host Nothing uses it and if we don't provide it we don't need getLocationHints --- src/allmydata/interfaces.py | 1 - src/allmydata/storage_client.py | 6 ------ src/allmydata/test/no_network.py | 3 --- src/allmydata/test/test_introducer.py | 2 -- 4 files changed, 12 deletions(-) diff --git a/src/allmydata/interfaces.py b/src/allmydata/interfaces.py index 49dcf7646..b9a757b08 100644 --- a/src/allmydata/interfaces.py +++ b/src/allmydata/interfaces.py @@ -521,7 +521,6 @@ class IStorageBroker(Interface): oldest_supported: the peer's oldest supported version, same rref: the RemoteReference, if connected, otherwise None - remote_host: the IAddress, if connected, otherwise None This method is intended for monitoring interfaces, such as a web page that describes connecting and connected peers. diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 823c15b7a..87560e51c 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -693,7 +693,6 @@ class NativeStorageServer(service.MultiService): @ivar nickname: the server's self-reported nickname (unicode), same @ivar rref: the RemoteReference, if connected, otherwise None - @ivar remote_host: the IAddress, if connected, otherwise None """ VERSION_DEFAULTS = UnicodeKeyDict({ @@ -719,7 +718,6 @@ class NativeStorageServer(service.MultiService): self.last_connect_time = None self.last_loss_time = None - self.remote_host = None self._rref = None self._is_connected = False self._reconnector = None @@ -828,8 +826,6 @@ class NativeStorageServer(service.MultiService): return None def get_announcement(self): return self.announcement - def get_remote_host(self): - return self.remote_host def get_connection_status(self): last_received = None @@ -877,7 +873,6 @@ class NativeStorageServer(service.MultiService): level=log.NOISY, parent=lp) self.last_connect_time = time.time() - self.remote_host = rref.getLocationHints() self._rref = rref self._is_connected = True rref.notifyOnDisconnect(self._lost) @@ -903,7 +898,6 @@ class NativeStorageServer(service.MultiService): # get_connected_servers() or get_servers_for_psi()) can continue to # use s.get_rref().callRemote() and not worry about it being None. self._is_connected = False - self.remote_host = None def stop_connecting(self): # used when this descriptor has been superceded by another diff --git a/src/allmydata/test/no_network.py b/src/allmydata/test/no_network.py index ad7659704..59ab807bb 100644 --- a/src/allmydata/test/no_network.py +++ b/src/allmydata/test/no_network.py @@ -89,9 +89,6 @@ class LocalWrapper(object): self.counter_by_methname = {} self._fireEventually = fireEventually - def getLocationHints(self): - return [] - def _clear_counters(self): self.counter_by_methname = {} diff --git a/src/allmydata/test/test_introducer.py b/src/allmydata/test/test_introducer.py index 2c5354fe0..5350335e0 100644 --- a/src/allmydata/test/test_introducer.py +++ b/src/allmydata/test/test_introducer.py @@ -744,8 +744,6 @@ class SystemTest(SystemTestMixin, AsyncTestCase): class FakeRemoteReference(object): def notifyOnDisconnect(self, *args, **kwargs): pass def getRemoteTubID(self): return "62ubehyunnyhzs7r6vdonnm2hpi52w6y" - def getLocationHints(self): return ["tcp:here.example.com:1234", - "tcp:there.example.com2345"] def getPeer(self): return address.IPv4Address("TCP", "remote.example.com", 3456) From 754b88a0b9cb9497d24c9f49430951d3bd452d2e Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 8 Dec 2020 13:47:53 -0500 Subject: [PATCH 22/22] Fix failing test. --- src/allmydata/test/common.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/common.py b/src/allmydata/test/common.py index 797e3dab9..f93272540 100644 --- a/src/allmydata/test/common.py +++ b/src/allmydata/test/common.py @@ -214,7 +214,7 @@ class UseNode(object): :ivar FilePath basedir: The base directory of the node. - :ivar bytes introducer_furl: The introducer furl with which to + :ivar str introducer_furl: The introducer furl with which to configure the client. :ivar dict[bytes, bytes] node_config: Configuration items for the *node* @@ -225,7 +225,8 @@ class UseNode(object): plugin_config = attr.ib() storage_plugin = attr.ib() basedir = attr.ib(validator=attr.validators.instance_of(FilePath)) - introducer_furl = attr.ib(validator=attr.validators.instance_of(bytes)) + introducer_furl = attr.ib(validator=attr.validators.instance_of(str), + converter=six.ensure_str) node_config = attr.ib(default=attr.Factory(dict)) config = attr.ib(default=None)