Merge pull request #914 from tahoe-lafs/3544.furls-not-bytes-maybe

Makes furls not bytes

Fixes ticket:3544
This commit is contained in:
Itamar Turner-Trauring 2020-12-09 09:20:32 -05:00 committed by GitHub
commit dab89f433c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 48 additions and 52 deletions

0
newsfragments/3544.minor Normal file
View File

View File

@ -1,5 +1,3 @@
from past.builtins import unicode
import re
from allmydata.crypto.util import remove_prefix
from allmydata.crypto import ed25519
@ -8,14 +6,12 @@ from allmydata.util import base32, rrefutil, jsonbytes as json
def get_tubid_string_from_ann(ann):
furl = ann.get("anonymous-storage-FURL") or ann.get("FURL")
if isinstance(furl, unicode):
furl = furl.encode("utf-8")
return get_tubid_string(furl)
def get_tubid_string(furl):
m = re.match(br'pb://(\w+)@', furl)
m = re.match(r'pb://(\w+)@', furl)
assert m
return m.group(1).lower()
return m.group(1).lower().encode("ascii")
def sign_to_foolscap(announcement, signing_key):

View File

@ -562,6 +562,7 @@ class _FoolscapStorage(object):
*nickname* is optional.
"""
furl = furl.encode("utf-8")
m = re.match(br'pb://(\w+)@', furl)
assert m, furl
tubid_s = m.group(1).lower()
@ -756,7 +757,7 @@ class NativeStorageServer(service.MultiService):
else:
return _FoolscapStorage.from_announcement(
self._server_id,
furl.encode("utf-8"),
furl,
ann,
storage_server,
)
@ -768,8 +769,6 @@ class NativeStorageServer(service.MultiService):
# Nope
pass
else:
if isinstance(furl, str):
furl = furl.encode("utf-8")
# See comment above for the _storage_from_foolscap_plugin case
# about passing in get_rref.
storage_server = _StorageServer(get_rref=self.get_rref)

View File

@ -214,7 +214,7 @@ class UseNode(object):
:ivar FilePath basedir: The base directory of the node.
:ivar bytes introducer_furl: The introducer furl with which to
:ivar str introducer_furl: The introducer furl with which to
configure the client.
:ivar dict[bytes, bytes] node_config: Configuration items for the *node*
@ -225,7 +225,8 @@ class UseNode(object):
plugin_config = attr.ib()
storage_plugin = attr.ib()
basedir = attr.ib(validator=attr.validators.instance_of(FilePath))
introducer_furl = attr.ib(validator=attr.validators.instance_of(bytes))
introducer_furl = attr.ib(validator=attr.validators.instance_of(str),
converter=six.ensure_str)
node_config = attr.ib(default=attr.Factory(dict))
config = attr.ib(default=None)

View File

@ -239,7 +239,7 @@ def make_peer(s, i):
peerid = base32.b2a(tagged_hash(b"peerid", b"%d" % i)[:20])
fss = FakeStorageServer(peerid, s)
ann = {
"anonymous-storage-FURL": b"pb://%s@nowhere/fake" % (peerid,),
"anonymous-storage-FURL": "pb://%s@nowhere/fake" % (str(peerid, "utf-8"),),
"permutation-seed-base32": peerid,
}
return Peer(peerid=peerid, storage_server=fss, announcement=ann)

View File

@ -156,7 +156,7 @@ class WebResultsRendering(unittest.TestCase):
for (key_s, binary_tubid, nickname) in servers:
server_id = key_s
tubid_b32 = base32.b2a(binary_tubid)
furl = b"pb://%s@nowhere/fake" % tubid_b32
furl = "pb://%s@nowhere/fake" % str(tubid_b32, "utf-8")
ann = { "version": 0,
"service-name": "storage",
"anonymous-storage-FURL": furl,

View File

@ -88,7 +88,7 @@ from .strategies import (
write_capabilities,
)
SOME_FURL = b"pb://abcde@nowhere/fake"
SOME_FURL = "pb://abcde@nowhere/fake"
BASECONFIG = "[client]\n"

View File

@ -216,9 +216,9 @@ class Client(AsyncTestCase):
def _received(key_s, ann):
announcements.append( (key_s, ann) )
ic1.subscribe_to("storage", _received)
furl1 = b"pb://62ubehyunnyhzs7r6vdonnm2hpi52w6y@127.0.0.1:36106/gydnp"
furl1a = b"pb://62ubehyunnyhzs7r6vdonnm2hpi52w6y@127.0.0.1:7777/gydnp"
furl2 = b"pb://ttwwooyunnyhzs7r6vdonnm2hpi52w6y@127.0.0.1:36106/ttwwoo"
furl1 = "pb://62ubehyunnyhzs7r6vdonnm2hpi52w6y@127.0.0.1:36106/gydnp"
furl1a = "pb://62ubehyunnyhzs7r6vdonnm2hpi52w6y@127.0.0.1:7777/gydnp"
furl2 = "pb://ttwwooyunnyhzs7r6vdonnm2hpi52w6y@127.0.0.1:36106/ttwwoo"
private_key, public_key = ed25519.create_signing_keypair()
public_key_str = ed25519.string_from_verifying_key(public_key)
@ -242,7 +242,7 @@ class Client(AsyncTestCase):
self.failUnlessEqual(len(announcements), 1)
key_s,ann = announcements[0]
self.failUnlessEqual(key_s, pubkey_s)
self.failUnlessEqual(ensure_binary(ann["anonymous-storage-FURL"]), furl1)
self.failUnlessEqual(ann["anonymous-storage-FURL"], furl1)
self.failUnlessEqual(ann["my-version"], "ver23")
d.addCallback(_then1)
@ -276,7 +276,7 @@ class Client(AsyncTestCase):
self.failUnlessEqual(len(announcements), 2)
key_s,ann = announcements[-1]
self.failUnlessEqual(key_s, pubkey_s)
self.failUnlessEqual(ensure_binary(ann["anonymous-storage-FURL"]), furl1)
self.failUnlessEqual(ann["anonymous-storage-FURL"], furl1)
self.failUnlessEqual(ann["my-version"], "ver24")
d.addCallback(_then3)
@ -288,7 +288,7 @@ class Client(AsyncTestCase):
self.failUnlessEqual(len(announcements), 3)
key_s,ann = announcements[-1]
self.failUnlessEqual(key_s, pubkey_s)
self.failUnlessEqual(ensure_binary(ann["anonymous-storage-FURL"]), furl1a)
self.failUnlessEqual(ann["anonymous-storage-FURL"], furl1a)
self.failUnlessEqual(ann["my-version"], "ver23")
d.addCallback(_then4)
@ -304,7 +304,7 @@ class Client(AsyncTestCase):
self.failUnlessEqual(len(announcements2), 1)
key_s,ann = announcements2[-1]
self.failUnlessEqual(key_s, pubkey_s)
self.failUnlessEqual(ensure_binary(ann["anonymous-storage-FURL"]), furl1a)
self.failUnlessEqual(ann["anonymous-storage-FURL"], furl1a)
self.failUnlessEqual(ann["my-version"], "ver23")
d.addCallback(_then5)
return d
@ -316,7 +316,7 @@ class Server(AsyncTestCase):
"introducer.furl", u"my_nickname",
"ver23", "oldest_version", realseq,
FilePath(self.mktemp()))
furl1 = b"pb://62ubehyunnyhzs7r6vdonnm2hpi52w6y@127.0.0.1:36106/gydnp"
furl1 = "pb://62ubehyunnyhzs7r6vdonnm2hpi52w6y@127.0.0.1:36106/gydnp"
private_key, _ = ed25519.create_signing_keypair()
@ -414,7 +414,7 @@ class Queue(SystemTestMixin, AsyncTestCase):
c = IntroducerClient(tub2, ifurl,
u"nickname", "version", "oldest", fakeseq,
FilePath(self.mktemp()))
furl1 = b"pb://onug64tu@127.0.0.1:123/short" # base32("short")
furl1 = "pb://onug64tu@127.0.0.1:123/short" # base32("short")
private_key, _ = ed25519.create_signing_keypair()
d = introducer.disownServiceParent()
@ -436,7 +436,7 @@ class Queue(SystemTestMixin, AsyncTestCase):
def _done(ign):
v = introducer.get_announcements()[0]
furl = v.announcement["anonymous-storage-FURL"]
self.failUnlessEqual(ensure_binary(furl), furl1)
self.failUnlessEqual(furl, furl1)
d.addCallback(_done)
# now let the ack get back
@ -462,7 +462,7 @@ class SystemTest(SystemTestMixin, AsyncTestCase):
iff = os.path.join(self.basedir, "introducer.furl")
tub = self.central_tub
ifurl = self.central_tub.registerReference(introducer, furlFile=iff)
self.introducer_furl = ifurl.encode("utf-8")
self.introducer_furl = ifurl
# we have 5 clients who publish themselves as storage servers, and a
# sixth which does which not. All 6 clients subscriber to hear about
@ -503,7 +503,7 @@ class SystemTest(SystemTestMixin, AsyncTestCase):
subscribing_clients.append(c)
expected_announcements[i] += 1 # all expect a 'storage' announcement
node_furl = tub.registerReference(Referenceable()).encode("utf-8")
node_furl = tub.registerReference(Referenceable())
private_key, public_key = ed25519.create_signing_keypair()
public_key_str = ed25519.string_from_verifying_key(public_key)
privkeys[i] = private_key
@ -520,7 +520,7 @@ class SystemTest(SystemTestMixin, AsyncTestCase):
if i == 2:
# also publish something that nobody cares about
boring_furl = tub.registerReference(Referenceable()).encode("utf-8")
boring_furl = tub.registerReference(Referenceable())
c.publish("boring", make_ann(boring_furl), private_key)
c.setServiceParent(self.parent)
@ -658,7 +658,7 @@ class SystemTest(SystemTestMixin, AsyncTestCase):
self.create_tub(self.central_portnum)
newfurl = self.central_tub.registerReference(self.the_introducer,
furlFile=iff)
assert ensure_binary(newfurl) == self.introducer_furl
assert newfurl == self.introducer_furl
d.addCallback(_restart_introducer_tub)
d.addCallback(_wait_for_connected)
@ -710,7 +710,7 @@ class SystemTest(SystemTestMixin, AsyncTestCase):
self.the_introducer = introducer
newfurl = self.central_tub.registerReference(self.the_introducer,
furlFile=iff)
assert ensure_binary(newfurl) == self.introducer_furl
assert newfurl == self.introducer_furl
d.addCallback(_restart_introducer)
d.addCallback(_wait_for_connected)
@ -754,7 +754,7 @@ class ClientInfo(AsyncTestCase):
client_v2 = IntroducerClient(tub, introducer_furl, NICKNAME % u"v2",
"my_version", "oldest",
fakeseq, FilePath(self.mktemp()))
#furl1 = b"pb://62ubehyunnyhzs7r6vdonnm2hpi52w6y@127.0.0.1:0/swissnum"
#furl1 = "pb://62ubehyunnyhzs7r6vdonnm2hpi52w6y@127.0.0.1:0/swissnum"
#ann_s = make_ann_t(client_v2, furl1, None, 10)
#introducer.remote_publish_v2(ann_s, Referenceable())
subscriber = FakeRemoteReference()
@ -775,7 +775,7 @@ class Announcements(AsyncTestCase):
client_v2 = IntroducerClient(tub, introducer_furl, u"nick-v2",
"my_version", "oldest",
fakeseq, FilePath(self.mktemp()))
furl1 = b"pb://62ubehyunnyhzs7r6vdonnm2hpi52w6y@127.0.0.1:0/swissnum"
furl1 = "pb://62ubehyunnyhzs7r6vdonnm2hpi52w6y@127.0.0.1:0/swissnum"
private_key, public_key = ed25519.create_signing_keypair()
public_key_str = remove_prefix(ed25519.string_from_verifying_key(public_key), b"pub-")
@ -790,7 +790,7 @@ class Announcements(AsyncTestCase):
self.failUnlessEqual(a[0].nickname, u"nick-v2")
self.failUnlessEqual(a[0].service_name, "storage")
self.failUnlessEqual(a[0].version, "my_version")
self.failUnlessEqual(ensure_binary(a[0].announcement["anonymous-storage-FURL"]), furl1)
self.failUnlessEqual(a[0].announcement["anonymous-storage-FURL"], furl1)
def _load_cache(self, cache_filepath):
with cache_filepath.open() as f:
@ -823,7 +823,7 @@ class Announcements(AsyncTestCase):
ic = c.introducer_clients[0]
private_key, public_key = ed25519.create_signing_keypair()
public_key_str = remove_prefix(ed25519.string_from_verifying_key(public_key), b"pub-")
furl1 = b"pb://onug64tu@127.0.0.1:123/short" # base32("short")
furl1 = "pb://onug64tu@127.0.0.1:123/short" # base32("short")
ann_t = make_ann_t(ic, furl1, private_key, 1)
ic.got_announcements([ann_t])
@ -834,12 +834,12 @@ class Announcements(AsyncTestCase):
self.failUnlessEqual(len(announcements), 1)
self.failUnlessEqual(ensure_binary(announcements[0]['key_s']), public_key_str)
ann = announcements[0]["ann"]
self.failUnlessEqual(ensure_binary(ann["anonymous-storage-FURL"]), furl1)
self.failUnlessEqual(ann["anonymous-storage-FURL"], furl1)
self.failUnlessEqual(ann["seqnum"], 1)
# a new announcement that replaces the first should replace the
# cached entry, not duplicate it
furl2 = furl1 + b"er"
furl2 = furl1 + "er"
ann_t2 = make_ann_t(ic, furl2, private_key, 2)
ic.got_announcements([ann_t2])
yield flushEventualQueue()
@ -847,14 +847,14 @@ class Announcements(AsyncTestCase):
self.failUnlessEqual(len(announcements), 1)
self.failUnlessEqual(ensure_binary(announcements[0]['key_s']), public_key_str)
ann = announcements[0]["ann"]
self.failUnlessEqual(ensure_binary(ann["anonymous-storage-FURL"]), furl2)
self.failUnlessEqual(ann["anonymous-storage-FURL"], furl2)
self.failUnlessEqual(ann["seqnum"], 2)
# but a third announcement with a different key should add to the
# cache
private_key2, public_key2 = ed25519.create_signing_keypair()
public_key_str2 = remove_prefix(ed25519.string_from_verifying_key(public_key2), b"pub-")
furl3 = b"pb://onug64tu@127.0.0.1:456/short"
furl3 = "pb://onug64tu@127.0.0.1:456/short"
ann_t3 = make_ann_t(ic, furl3, private_key2, 1)
ic.got_announcements([ann_t3])
yield flushEventualQueue()
@ -864,7 +864,7 @@ class Announcements(AsyncTestCase):
self.failUnlessEqual(set([public_key_str, public_key_str2]),
set([ensure_binary(a["key_s"]) for a in announcements]))
self.failUnlessEqual(set([furl2, furl3]),
set([ensure_binary(a["ann"]["anonymous-storage-FURL"])
set([a["ann"]["anonymous-storage-FURL"]
for a in announcements]))
# test loading
@ -880,9 +880,9 @@ class Announcements(AsyncTestCase):
yield flushEventualQueue()
self.failUnless(public_key_str in announcements)
self.failUnlessEqual(ensure_binary(announcements[public_key_str]["anonymous-storage-FURL"]),
self.failUnlessEqual(announcements[public_key_str]["anonymous-storage-FURL"],
furl2)
self.failUnlessEqual(ensure_binary(announcements[public_key_str2]["anonymous-storage-FURL"]),
self.failUnlessEqual(announcements[public_key_str2]["anonymous-storage-FURL"],
furl3)
c2 = yield create_client(basedir.path)
@ -997,10 +997,10 @@ class DecodeFurl(SyncTestCase):
def test_decode(self):
# make sure we have a working base64.b32decode. The one in
# python2.4.[01] was broken.
furl = b'pb://t5g7egomnnktbpydbuijt6zgtmw4oqi5@127.0.0.1:51857/hfzv36i'
m = re.match(br'pb://(\w+)@', furl)
furl = 'pb://t5g7egomnnktbpydbuijt6zgtmw4oqi5@127.0.0.1:51857/hfzv36i'
m = re.match(r'pb://(\w+)@', furl)
assert m
nodeid = b32decode(m.group(1).upper())
nodeid = b32decode(m.group(1).upper().encode("ascii"))
self.failUnlessEqual(nodeid, b"\x9fM\xf2\x19\xcckU0\xbf\x03\r\x10\x99\xfb&\x9b-\xc7A\x1d")
class Signatures(SyncTestCase):
@ -1041,7 +1041,7 @@ class Signatures(SyncTestCase):
mock_tub = Mock()
ic = IntroducerClient(
mock_tub,
b"pb://",
"pb://",
u"fake_nick",
"0.0.0",
"1.2.3",

View File

@ -103,7 +103,7 @@ from allmydata.interfaces import (
IStorageServer,
)
SOME_FURL = b"pb://abcde@nowhere/fake"
SOME_FURL = "pb://abcde@nowhere/fake"
class NativeStorageServerWithVersion(NativeStorageServer):
def __init__(self, version):
@ -310,7 +310,7 @@ class PluginMatchedAnnouncement(SyncTestCase):
# notice how the announcement is for a different storage plugin
# than the one that is enabled.
u"name": u"tahoe-lafs-dummy-v2",
u"storage-server-FURL": SOME_FURL.decode("ascii"),
u"storage-server-FURL": SOME_FURL,
}],
}
self.publish(server_id, ann, self.introducer_client)
@ -338,7 +338,7 @@ class PluginMatchedAnnouncement(SyncTestCase):
u"storage-options": [{
# and this announcement is for a plugin with a matching name
u"name": plugin_name,
u"storage-server-FURL": SOME_FURL.decode("ascii"),
u"storage-server-FURL": SOME_FURL,
}],
}
self.publish(server_id, ann, self.introducer_client)
@ -389,7 +389,7 @@ class PluginMatchedAnnouncement(SyncTestCase):
u"storage-options": [{
# and this announcement is for a plugin with a matching name
u"name": plugin_name,
u"storage-server-FURL": SOME_FURL.decode("ascii"),
u"storage-server-FURL": SOME_FURL,
}],
}
self.publish(server_id, ann, self.introducer_client)
@ -594,7 +594,7 @@ storage:
ann:
anonymous-storage-FURL: {furl}
permutation-seed-base32: aaaaaaaaaaaaaaaaaaaaaaaa
""".format(furl=SOME_FURL.decode("utf-8"))
""".format(furl=SOME_FURL)
servers = yamlutil.safe_load(servers_yaml)
permseed = base32.a2b(b"aaaaaaaaaaaaaaaaaaaaaaaa")
broker.set_static_servers(servers["storage"])
@ -610,7 +610,7 @@ storage:
ann2 = {
"service-name": "storage",
"anonymous-storage-FURL": "pb://{}@nowhere/fake2".format(base32.b2a(b"1")),
"anonymous-storage-FURL": "pb://{}@nowhere/fake2".format(str(base32.b2a(b"1"), "utf-8")),
"permutation-seed-base32": "bbbbbbbbbbbbbbbbbbbbbbbb",
}
broker._got_announcement(key_s, ann2)
@ -694,7 +694,7 @@ storage:
}
def add_one_server(x):
data["anonymous-storage-FURL"] = b"pb://%s@spy:nowhere/fake" % (base32.b2a(b"%d" % x),)
data["anonymous-storage-FURL"] = "pb://%s@spy:nowhere/fake" % (str(base32.b2a(b"%d" % x), "ascii"),)
tub = new_tub()
connects = []
spy = SpyHandler(connects)

View File

@ -239,7 +239,7 @@ class FakeClient(object):
node_config=EMPTY_CLIENT_CONFIG,
)
for (serverid, rref) in servers:
ann = {"anonymous-storage-FURL": b"pb://%s@nowhere/fake" % base32.b2a(serverid),
ann = {"anonymous-storage-FURL": "pb://%s@nowhere/fake" % str(base32.b2a(serverid), "ascii"),
"permutation-seed-base32": base32.b2a(serverid) }
self.storage_broker.test_add_rref(serverid, rref, ann)
self.last_servers = [s[1] for s in servers]