ExternalVendorCode/tahoe-lafs
1
0
Fork 0
mirror of https://github.com/tahoe-lafs/tahoe-lafs.git synced 2025-04-13 05:43:03 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/master' into 3522.test_alias-no-mock

Browse Source
This commit is contained in:
Jean-Paul Calderone 2020-12-10 07:19:43 -05:00
commit 4ac95a1ede
16 changed files with 225 additions and 93 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

0
newsfragments/3521.minor Normal file
View File

0
newsfragments/3544.minor Normal file
View File

0
newsfragments/3546.minor Normal file
View File

1
src/allmydata/interfaces.py
View File

@ -521,7 +521,6 @@ class IStorageBroker(Interface):
oldest_supported: the peer's oldest supported version, same
rref: the RemoteReference, if connected, otherwise None
remote_host: the IAddress, if connected, otherwise None
This method is intended for monitoring interfaces, such as a web page
that describes connecting and connected peers.

30
src/allmydata/introducer/client.py
View File

@ -1,4 +1,16 @@
from past.builtins import unicode, long
"""
Ported to Python 3.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
from future.utils import PY2
if PY2:
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
from past.builtins import long
from six import ensure_text
import time
@ -27,11 +39,11 @@ class IntroducerClient(service.Service, Referenceable):
nickname, my_version, oldest_supported,
sequencer, cache_filepath):
self._tub = tub
if isinstance(introducer_furl, unicode):
if isinstance(introducer_furl, str):
introducer_furl = introducer_furl.encode("utf-8")
self.introducer_furl = introducer_furl
assert type(nickname) is unicode
assert isinstance(nickname, str)
self._nickname = nickname
self._my_version = my_version
self._oldest_supported = oldest_supported
@ -114,7 +126,7 @@ class IntroducerClient(service.Service, Referenceable):
def _save_announcements(self):
announcements = []
for _, value in self._inbound_announcements.items():
for value in self._inbound_announcements.values():
ann, key_s, time_stamp = value
# On Python 2, bytes strings are encoded into YAML Unicode strings.
# On Python 3, bytes are encoded as YAML bytes. To minimize
@ -125,7 +137,7 @@ class IntroducerClient(service.Service, Referenceable):
}
announcements.append(server_params)
announcement_cache_yaml = yamlutil.safe_dump(announcements)
if isinstance(announcement_cache_yaml, unicode):
if isinstance(announcement_cache_yaml, str):
announcement_cache_yaml = announcement_cache_yaml.encode("utf-8")
self._cache_filepath.setContent(announcement_cache_yaml)
@ -170,7 +182,7 @@ class IntroducerClient(service.Service, Referenceable):
self._local_subscribers.append( (service_name,cb,args,kwargs) )
self._subscribed_service_names.add(service_name)
self._maybe_subscribe()
for index,(ann,key_s,when) in self._inbound_announcements.items():
for index,(ann,key_s,when) in list(self._inbound_announcements.items()):
precondition(isinstance(key_s, bytes), key_s)
servicename = index[0]
if servicename == service_name:
@ -215,7 +227,7 @@ class IntroducerClient(service.Service, Referenceable):
self._outbound_announcements[service_name] = ann_d
# publish all announcements with the new seqnum and nonce
for service_name,ann_d in self._outbound_announcements.items():
for service_name,ann_d in list(self._outbound_announcements.items()):
ann_d["seqnum"] = current_seqnum
ann_d["nonce"] = current_nonce
ann_t = sign_to_foolscap(ann_d, signing_key)
@ -227,7 +239,7 @@ class IntroducerClient(service.Service, Referenceable):
self.log("want to publish, but no introducer yet", level=log.NOISY)
return
# this re-publishes everything. The Introducer ignores duplicates
for ann_t in self._published_announcements.values():
for ann_t in list(self._published_announcements.values()):
self._debug_counts["outbound_message"] += 1
self._debug_outstanding += 1
d = self._publisher.callRemote("publish_v2", ann_t, self._canary)
@ -267,7 +279,7 @@ class IntroducerClient(service.Service, Referenceable):
return
# for ASCII values, simplejson might give us unicode *or* bytes
if "nickname" in ann and isinstance(ann["nickname"], bytes):
ann["nickname"] = unicode(ann["nickname"])
ann["nickname"] = str(ann["nickname"])
nick_s = ann.get("nickname",u"").encode("utf-8")
lp2 = self.log(format="announcement for nickname '%(nick)s', service=%(svc)s: %(ann)s",
nick=nick_s, svc=service_name, ann=ann, umid="BoKEag")

18
src/allmydata/introducer/common.py
View File

@ -1,4 +1,14 @@
from past.builtins import unicode
"""
Ported to Python 3.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
from future.utils import PY2
if PY2:
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
import re
from allmydata.crypto.util import remove_prefix
@ -8,14 +18,12 @@ from allmydata.util import base32, rrefutil, jsonbytes as json
def get_tubid_string_from_ann(ann):
furl = ann.get("anonymous-storage-FURL") or ann.get("FURL")
if isinstance(furl, unicode):
furl = furl.encode("utf-8")
return get_tubid_string(furl)
def get_tubid_string(furl):
m = re.match(br'pb://(\w+)@', furl)
m = re.match(r'pb://(\w+)@', furl)
assert m
return m.group(1).lower()
return m.group(1).lower().encode("ascii")
def sign_to_foolscap(announcement, signing_key):

33
src/allmydata/introducer/server.py
View File

@ -1,5 +1,18 @@
"""
Ported to Python 3.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
from future.utils import PY2
if PY2:
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
from past.builtins import long
from six import ensure_str, ensure_text
from six import ensure_text
import time, os.path, textwrap
from zope.interface import implementer
@ -157,7 +170,7 @@ class IntroducerService(service.MultiService, Referenceable):
# 'subscriber_info' is a dict, provided directly by v2 clients. The
# expected keys are: version, nickname, app-versions, my-version,
# oldest-supported
self._subscribers = {}
self._subscribers = dictutil.UnicodeKeyDict({})
self._debug_counts = {"inbound_message": 0,
"inbound_duplicate": 0,
@ -181,7 +194,7 @@ class IntroducerService(service.MultiService, Referenceable):
def get_announcements(self):
"""Return a list of AnnouncementDescriptor for all announcements"""
announcements = []
for (index, (_, canary, ann, when)) in self._announcements.items():
for (index, (_, canary, ann, when)) in list(self._announcements.items()):
ad = AnnouncementDescriptor(when, index, canary, ann)
announcements.append(ad)
return announcements
@ -189,8 +202,8 @@ class IntroducerService(service.MultiService, Referenceable):
def get_subscribers(self):
"""Return a list of SubscriberDescriptor objects for all subscribers"""
s = []
for service_name, subscriptions in self._subscribers.items():
for rref,(subscriber_info,when) in subscriptions.items():
for service_name, subscriptions in list(self._subscribers.items()):
for rref,(subscriber_info,when) in list(subscriptions.items()):
# note that if the subscriber didn't do Tub.setLocation,
# tubid will be None. Also, subscribers do not tell us which
# pubkey they use; only publishers do that.
@ -281,7 +294,7 @@ class IntroducerService(service.MultiService, Referenceable):
def remote_subscribe_v2(self, subscriber, service_name, subscriber_info):
self.log("introducer: subscription[%s] request at %s"
% (service_name, subscriber), umid="U3uzLg")
service_name = ensure_str(service_name)
service_name = ensure_text(service_name)
subscriber_info = dictutil.UnicodeKeyDict({
ensure_text(k): v for (k, v) in subscriber_info.items()
})
@ -307,11 +320,11 @@ class IntroducerService(service.MultiService, Referenceable):
subscribers.pop(subscriber, None)
subscriber.notifyOnDisconnect(_remove)
# Make sure types are correct:
for k in self._announcements:
assert isinstance(k[0], type(service_name))
# now tell them about any announcements they're interested in
assert {type(service_name)}.issuperset(
set(type(k[0]) for k in self._announcements)), (
service_name, self._announcements.keys()
)
announcements = set( [ ann_t
for idx,(ann_t,canary,ann,when)
in self._announcements.items()

17
src/allmydata/storage_client.py
View File

@ -154,6 +154,9 @@ class StorageFarmBroker(service.MultiService):
I'm also responsible for subscribing to the IntroducerClient to find out
about new servers as they are announced by the Introducer.
:ivar _tub_maker: A one-argument callable which accepts a dictionary of
"handler overrides" and returns a ``foolscap.api.Tub``.
:ivar StorageClientConfig storage_client_config: Values from the node
configuration file relating to storage behavior.
"""
@ -558,7 +561,11 @@ class _FoolscapStorage(object):
}
*nickname* is optional.
The furl will be a Unicode string on Python 3; on Python 2 it will be
either a native (bytes) string or a Unicode string.
"""
furl = furl.encode("utf-8")
m = re.match(br'pb://(\w+)@', furl)
assert m, furl
tubid_s = m.group(1).lower()
@ -690,7 +697,6 @@ class NativeStorageServer(service.MultiService):
@ivar nickname: the server's self-reported nickname (unicode), same
@ivar rref: the RemoteReference, if connected, otherwise None
@ivar remote_host: the IAddress, if connected, otherwise None
"""
VERSION_DEFAULTS = UnicodeKeyDict({
@ -716,7 +722,6 @@ class NativeStorageServer(service.MultiService):
self.last_connect_time = None
self.last_loss_time = None
self.remote_host = None
self._rref = None
self._is_connected = False
self._reconnector = None
@ -755,7 +760,7 @@ class NativeStorageServer(service.MultiService):
else:
return _FoolscapStorage.from_announcement(
self._server_id,
furl.encode("utf-8"),
furl,
ann,
storage_server,
)
@ -767,8 +772,6 @@ class NativeStorageServer(service.MultiService):
# Nope
pass
else:
if isinstance(furl, str):
furl = furl.encode("utf-8")
# See comment above for the _storage_from_foolscap_plugin case
# about passing in get_rref.
storage_server = _StorageServer(get_rref=self.get_rref)
@ -825,8 +828,6 @@ class NativeStorageServer(service.MultiService):
return None
def get_announcement(self):
return self.announcement
def get_remote_host(self):
return self.remote_host
def get_connection_status(self):
last_received = None
@ -874,7 +875,6 @@ class NativeStorageServer(service.MultiService):
level=log.NOISY, parent=lp)
self.last_connect_time = time.time()
self.remote_host = rref.getLocationHints()
self._rref = rref
self._is_connected = True
rref.notifyOnDisconnect(self._lost)
@ -900,7 +900,6 @@ class NativeStorageServer(service.MultiService):
# get_connected_servers() or get_servers_for_psi()) can continue to
# use s.get_rref().callRemote() and not worry about it being None.
self._is_connected = False
self.remote_host = None
def stop_connecting(self):
# used when this descriptor has been superceded by another

5
src/allmydata/test/common.py
View File

@ -214,7 +214,7 @@ class UseNode(object):
:ivar FilePath basedir: The base directory of the node.
:ivar bytes introducer_furl: The introducer furl with which to
:ivar str introducer_furl: The introducer furl with which to
configure the client.
:ivar dict[bytes, bytes] node_config: Configuration items for the *node*
@ -225,7 +225,8 @@ class UseNode(object):
plugin_config = attr.ib()
storage_plugin = attr.ib()
basedir = attr.ib(validator=attr.validators.instance_of(FilePath))
introducer_furl = attr.ib(validator=attr.validators.instance_of(bytes))
introducer_furl = attr.ib(validator=attr.validators.instance_of(str),
converter=six.ensure_str)
node_config = attr.ib(default=attr.Factory(dict))
config = attr.ib(default=None)

2
src/allmydata/test/mutable/util.py
View File

@ -239,7 +239,7 @@ def make_peer(s, i):
peerid = base32.b2a(tagged_hash(b"peerid", b"%d" % i)[:20])
fss = FakeStorageServer(peerid, s)
ann = {
"anonymous-storage-FURL": b"pb://%s@nowhere/fake" % (peerid,),
"anonymous-storage-FURL": "pb://%s@nowhere/fake" % (str(peerid, "utf-8"),),
"permutation-seed-base32": peerid,
}
return Peer(peerid=peerid, storage_server=fss, announcement=ann)

2
src/allmydata/test/test_checker.py
View File

@ -156,7 +156,7 @@ class WebResultsRendering(unittest.TestCase):
for (key_s, binary_tubid, nickname) in servers:
server_id = key_s
tubid_b32 = base32.b2a(binary_tubid)
furl = b"pb://%s@nowhere/fake" % tubid_b32
furl = "pb://%s@nowhere/fake" % str(tubid_b32, "utf-8")
ann = { "version": 0,
"service-name": "storage",
"anonymous-storage-FURL": furl,

2
src/allmydata/test/test_client.py
View File

@ -88,7 +88,7 @@ from .strategies import (
write_capabilities,
)
SOME_FURL = b"pb://abcde@nowhere/fake"
SOME_FURL = "pb://abcde@nowhere/fake"
BASECONFIG = "[client]\n"

62
src/allmydata/test/test_introducer.py
View File

@ -216,9 +216,9 @@ class Client(AsyncTestCase):
def _received(key_s, ann):
announcements.append( (key_s, ann) )
ic1.subscribe_to("storage", _received)
furl1 = b"pb://62ubehyunnyhzs7r6vdonnm2hpi52w6y@127.0.0.1:36106/gydnp"
furl1a = b"pb://62ubehyunnyhzs7r6vdonnm2hpi52w6y@127.0.0.1:7777/gydnp"
furl2 = b"pb://ttwwooyunnyhzs7r6vdonnm2hpi52w6y@127.0.0.1:36106/ttwwoo"
furl1 = "pb://62ubehyunnyhzs7r6vdonnm2hpi52w6y@127.0.0.1:36106/gydnp"
furl1a = "pb://62ubehyunnyhzs7r6vdonnm2hpi52w6y@127.0.0.1:7777/gydnp"
furl2 = "pb://ttwwooyunnyhzs7r6vdonnm2hpi52w6y@127.0.0.1:36106/ttwwoo"
private_key, public_key = ed25519.create_signing_keypair()
public_key_str = ed25519.string_from_verifying_key(public_key)
@ -242,7 +242,7 @@ class Client(AsyncTestCase):
self.failUnlessEqual(len(announcements), 1)
key_s,ann = announcements[0]
self.failUnlessEqual(key_s, pubkey_s)
self.failUnlessEqual(ensure_binary(ann["anonymous-storage-FURL"]), furl1)
self.failUnlessEqual(ann["anonymous-storage-FURL"], furl1)
self.failUnlessEqual(ann["my-version"], "ver23")
d.addCallback(_then1)
@ -276,7 +276,7 @@ class Client(AsyncTestCase):
self.failUnlessEqual(len(announcements), 2)
key_s,ann = announcements[-1]
self.failUnlessEqual(key_s, pubkey_s)
self.failUnlessEqual(ensure_binary(ann["anonymous-storage-FURL"]), furl1)
self.failUnlessEqual(ann["anonymous-storage-FURL"], furl1)
self.failUnlessEqual(ann["my-version"], "ver24")
d.addCallback(_then3)
@ -288,7 +288,7 @@ class Client(AsyncTestCase):
self.failUnlessEqual(len(announcements), 3)
key_s,ann = announcements[-1]
self.failUnlessEqual(key_s, pubkey_s)
self.failUnlessEqual(ensure_binary(ann["anonymous-storage-FURL"]), furl1a)
self.failUnlessEqual(ann["anonymous-storage-FURL"], furl1a)
self.failUnlessEqual(ann["my-version"], "ver23")
d.addCallback(_then4)
@ -304,7 +304,7 @@ class Client(AsyncTestCase):
self.failUnlessEqual(len(announcements2), 1)
key_s,ann = announcements2[-1]
self.failUnlessEqual(key_s, pubkey_s)
self.failUnlessEqual(ensure_binary(ann["anonymous-storage-FURL"]), furl1a)
self.failUnlessEqual(ann["anonymous-storage-FURL"], furl1a)
self.failUnlessEqual(ann["my-version"], "ver23")
d.addCallback(_then5)
return d
@ -316,7 +316,7 @@ class Server(AsyncTestCase):
"introducer.furl", u"my_nickname",
"ver23", "oldest_version", realseq,
FilePath(self.mktemp()))
furl1 = b"pb://62ubehyunnyhzs7r6vdonnm2hpi52w6y@127.0.0.1:36106/gydnp"
furl1 = "pb://62ubehyunnyhzs7r6vdonnm2hpi52w6y@127.0.0.1:36106/gydnp"
private_key, _ = ed25519.create_signing_keypair()
@ -414,7 +414,7 @@ class Queue(SystemTestMixin, AsyncTestCase):
c = IntroducerClient(tub2, ifurl,
u"nickname", "version", "oldest", fakeseq,
FilePath(self.mktemp()))
furl1 = b"pb://onug64tu@127.0.0.1:123/short" # base32("short")
furl1 = "pb://onug64tu@127.0.0.1:123/short" # base32("short")
private_key, _ = ed25519.create_signing_keypair()
d = introducer.disownServiceParent()
@ -436,7 +436,7 @@ class Queue(SystemTestMixin, AsyncTestCase):
def _done(ign):
v = introducer.get_announcements()[0]
furl = v.announcement["anonymous-storage-FURL"]
self.failUnlessEqual(ensure_binary(furl), furl1)
self.failUnlessEqual(furl, furl1)
d.addCallback(_done)
# now let the ack get back
@ -462,7 +462,7 @@ class SystemTest(SystemTestMixin, AsyncTestCase):
iff = os.path.join(self.basedir, "introducer.furl")
tub = self.central_tub
ifurl = self.central_tub.registerReference(introducer, furlFile=iff)
self.introducer_furl = ifurl.encode("utf-8")
self.introducer_furl = ifurl
# we have 5 clients who publish themselves as storage servers, and a
# sixth which does which not. All 6 clients subscriber to hear about
@ -503,7 +503,7 @@ class SystemTest(SystemTestMixin, AsyncTestCase):
subscribing_clients.append(c)
expected_announcements[i] += 1 # all expect a 'storage' announcement
node_furl = tub.registerReference(Referenceable()).encode("utf-8")
node_furl = tub.registerReference(Referenceable())
private_key, public_key = ed25519.create_signing_keypair()
public_key_str = ed25519.string_from_verifying_key(public_key)
privkeys[i] = private_key
@ -520,7 +520,7 @@ class SystemTest(SystemTestMixin, AsyncTestCase):
if i == 2:
# also publish something that nobody cares about
boring_furl = tub.registerReference(Referenceable()).encode("utf-8")
boring_furl = tub.registerReference(Referenceable())
c.publish("boring", make_ann(boring_furl), private_key)
c.setServiceParent(self.parent)
@ -658,7 +658,7 @@ class SystemTest(SystemTestMixin, AsyncTestCase):
self.create_tub(self.central_portnum)
newfurl = self.central_tub.registerReference(self.the_introducer,
furlFile=iff)
assert ensure_binary(newfurl) == self.introducer_furl
assert newfurl == self.introducer_furl
d.addCallback(_restart_introducer_tub)
d.addCallback(_wait_for_connected)
@ -710,7 +710,7 @@ class SystemTest(SystemTestMixin, AsyncTestCase):
self.the_introducer = introducer
newfurl = self.central_tub.registerReference(self.the_introducer,
furlFile=iff)
assert ensure_binary(newfurl) == self.introducer_furl
assert newfurl == self.introducer_furl
d.addCallback(_restart_introducer)
d.addCallback(_wait_for_connected)
@ -744,8 +744,6 @@ class SystemTest(SystemTestMixin, AsyncTestCase):
class FakeRemoteReference(object):
def notifyOnDisconnect(self, *args, **kwargs): pass
def getRemoteTubID(self): return "62ubehyunnyhzs7r6vdonnm2hpi52w6y"
def getLocationHints(self): return ["tcp:here.example.com:1234",
"tcp:there.example.com2345"]
def getPeer(self): return address.IPv4Address("TCP", "remote.example.com",
3456)
@ -756,7 +754,7 @@ class ClientInfo(AsyncTestCase):
client_v2 = IntroducerClient(tub, introducer_furl, NICKNAME % u"v2",
"my_version", "oldest",
fakeseq, FilePath(self.mktemp()))
#furl1 = b"pb://62ubehyunnyhzs7r6vdonnm2hpi52w6y@127.0.0.1:0/swissnum"
#furl1 = "pb://62ubehyunnyhzs7r6vdonnm2hpi52w6y@127.0.0.1:0/swissnum"
#ann_s = make_ann_t(client_v2, furl1, None, 10)
#introducer.remote_publish_v2(ann_s, Referenceable())
subscriber = FakeRemoteReference()
@ -777,7 +775,7 @@ class Announcements(AsyncTestCase):
client_v2 = IntroducerClient(tub, introducer_furl, u"nick-v2",
"my_version", "oldest",
fakeseq, FilePath(self.mktemp()))
furl1 = b"pb://62ubehyunnyhzs7r6vdonnm2hpi52w6y@127.0.0.1:0/swissnum"
furl1 = "pb://62ubehyunnyhzs7r6vdonnm2hpi52w6y@127.0.0.1:0/swissnum"
private_key, public_key = ed25519.create_signing_keypair()
public_key_str = remove_prefix(ed25519.string_from_verifying_key(public_key), b"pub-")
@ -792,7 +790,7 @@ class Announcements(AsyncTestCase):
self.failUnlessEqual(a[0].nickname, u"nick-v2")
self.failUnlessEqual(a[0].service_name, "storage")
self.failUnlessEqual(a[0].version, "my_version")
self.failUnlessEqual(ensure_binary(a[0].announcement["anonymous-storage-FURL"]), furl1)
self.failUnlessEqual(a[0].announcement["anonymous-storage-FURL"], furl1)
def _load_cache(self, cache_filepath):
with cache_filepath.open() as f:
@ -825,7 +823,7 @@ class Announcements(AsyncTestCase):
ic = c.introducer_clients[0]
private_key, public_key = ed25519.create_signing_keypair()
public_key_str = remove_prefix(ed25519.string_from_verifying_key(public_key), b"pub-")
furl1 = b"pb://onug64tu@127.0.0.1:123/short" # base32("short")
furl1 = "pb://onug64tu@127.0.0.1:123/short" # base32("short")
ann_t = make_ann_t(ic, furl1, private_key, 1)
ic.got_announcements([ann_t])
@ -836,12 +834,12 @@ class Announcements(AsyncTestCase):
self.failUnlessEqual(len(announcements), 1)
self.failUnlessEqual(ensure_binary(announcements[0]['key_s']), public_key_str)
ann = announcements[0]["ann"]
self.failUnlessEqual(ensure_binary(ann["anonymous-storage-FURL"]), furl1)
self.failUnlessEqual(ann["anonymous-storage-FURL"], furl1)
self.failUnlessEqual(ann["seqnum"], 1)
# a new announcement that replaces the first should replace the
# cached entry, not duplicate it
furl2 = furl1 + b"er"
furl2 = furl1 + "er"
ann_t2 = make_ann_t(ic, furl2, private_key, 2)
ic.got_announcements([ann_t2])
yield flushEventualQueue()
@ -849,14 +847,14 @@ class Announcements(AsyncTestCase):
self.failUnlessEqual(len(announcements), 1)
self.failUnlessEqual(ensure_binary(announcements[0]['key_s']), public_key_str)
ann = announcements[0]["ann"]
self.failUnlessEqual(ensure_binary(ann["anonymous-storage-FURL"]), furl2)
self.failUnlessEqual(ann["anonymous-storage-FURL"], furl2)
self.failUnlessEqual(ann["seqnum"], 2)
# but a third announcement with a different key should add to the
# cache
private_key2, public_key2 = ed25519.create_signing_keypair()
public_key_str2 = remove_prefix(ed25519.string_from_verifying_key(public_key2), b"pub-")
furl3 = b"pb://onug64tu@127.0.0.1:456/short"
furl3 = "pb://onug64tu@127.0.0.1:456/short"
ann_t3 = make_ann_t(ic, furl3, private_key2, 1)
ic.got_announcements([ann_t3])
yield flushEventualQueue()
@ -866,7 +864,7 @@ class Announcements(AsyncTestCase):
self.failUnlessEqual(set([public_key_str, public_key_str2]),
set([ensure_binary(a["key_s"]) for a in announcements]))
self.failUnlessEqual(set([furl2, furl3]),
set([ensure_binary(a["ann"]["anonymous-storage-FURL"])
set([a["ann"]["anonymous-storage-FURL"]
for a in announcements]))
# test loading
@ -882,9 +880,9 @@ class Announcements(AsyncTestCase):
yield flushEventualQueue()
self.failUnless(public_key_str in announcements)
self.failUnlessEqual(ensure_binary(announcements[public_key_str]["anonymous-storage-FURL"]),
self.failUnlessEqual(announcements[public_key_str]["anonymous-storage-FURL"],
furl2)
self.failUnlessEqual(ensure_binary(announcements[public_key_str2]["anonymous-storage-FURL"]),
self.failUnlessEqual(announcements[public_key_str2]["anonymous-storage-FURL"],
furl3)
c2 = yield create_client(basedir.path)
@ -999,10 +997,10 @@ class DecodeFurl(SyncTestCase):
def test_decode(self):
# make sure we have a working base64.b32decode. The one in
# python2.4.[01] was broken.
furl = b'pb://t5g7egomnnktbpydbuijt6zgtmw4oqi5@127.0.0.1:51857/hfzv36i'
m = re.match(br'pb://(\w+)@', furl)
furl = 'pb://t5g7egomnnktbpydbuijt6zgtmw4oqi5@127.0.0.1:51857/hfzv36i'
m = re.match(r'pb://(\w+)@', furl)
assert m
nodeid = b32decode(m.group(1).upper())
nodeid = b32decode(m.group(1).upper().encode("ascii"))
self.failUnlessEqual(nodeid, b"\x9fM\xf2\x19\xcckU0\xbf\x03\r\x10\x99\xfb&\x9b-\xc7A\x1d")
class Signatures(SyncTestCase):
@ -1043,7 +1041,7 @@ class Signatures(SyncTestCase):
mock_tub = Mock()
ic = IntroducerClient(
mock_tub,
b"pb://",
"pb://",
u"fake_nick",
"0.0.0",
"1.2.3",

141
src/allmydata/test/test_storage_client.py
View File

@ -17,7 +17,6 @@ from json import (
)
import hashlib
from mock import Mock
from fixtures import (
TempDir,
)
@ -44,12 +43,20 @@ from hyperlink import (
URL,
)
import attr
from twisted.internet.interfaces import (
IStreamClientEndpoint,
)
from twisted.application.service import (
Service,
)
from twisted.trial import unittest
from twisted.internet.defer import succeed, inlineCallbacks
from twisted.internet.defer import (
Deferred,
inlineCallbacks,
)
from twisted.python.filepath import (
FilePath,
)
@ -57,7 +64,11 @@ from twisted.python.filepath import (
from foolscap.api import (
Tub,
)
from foolscap.ipb import (
IConnectionHintHandler,
)
from .no_network import LocalWrapper
from .common import (
EMPTY_CLIENT_CONFIG,
SyncTestCase,
@ -65,6 +76,7 @@ from .common import (
UseTestPlugins,
UseNode,
SameProcessStreamEndpointAssigner,
MemoryIntroducerClient,
)
from .common_web import (
do_http,
@ -83,12 +95,15 @@ from allmydata.storage_client import (
_FoolscapStorage,
_NullStorage,
)
from ..storage.server import (
StorageServer,
)
from allmydata.interfaces import (
IConnectionStatus,
IStorageServer,
)
SOME_FURL = b"pb://abcde@nowhere/fake"
SOME_FURL = "pb://abcde@nowhere/fake"
class NativeStorageServerWithVersion(NativeStorageServer):
def __init__(self, version):
@ -295,7 +310,7 @@ class PluginMatchedAnnouncement(SyncTestCase):
# notice how the announcement is for a different storage plugin
# than the one that is enabled.
u"name": u"tahoe-lafs-dummy-v2",
u"storage-server-FURL": SOME_FURL.decode("ascii"),
u"storage-server-FURL": SOME_FURL,
}],
}
self.publish(server_id, ann, self.introducer_client)
@ -323,7 +338,7 @@ class PluginMatchedAnnouncement(SyncTestCase):
u"storage-options": [{
# and this announcement is for a plugin with a matching name
u"name": plugin_name,
u"storage-server-FURL": SOME_FURL.decode("ascii"),
u"storage-server-FURL": SOME_FURL,
}],
}
self.publish(server_id, ann, self.introducer_client)
@ -374,7 +389,7 @@ class PluginMatchedAnnouncement(SyncTestCase):
u"storage-options": [{
# and this announcement is for a plugin with a matching name
u"name": plugin_name,
u"storage-server-FURL": SOME_FURL.decode("ascii"),
u"storage-server-FURL": SOME_FURL,
}],
}
self.publish(server_id, ann, self.introducer_client)
@ -505,14 +520,68 @@ class StoragePluginWebPresence(AsyncTestCase):
)
def make_broker(tub_maker=lambda h: Mock()):
_aCertPEM = Tub().myCertificate.dumpPEM()
def new_tub():
"""
Make a new ``Tub`` with a hard-coded private key.
"""
# Use a private key / certificate generated by Tub how it wants. But just
# re-use the same one every time so we don't waste a lot of time
# generating them over and over in the tests.
return Tub(certData=_aCertPEM)
def make_broker(tub_maker=None):
"""
Create a ``StorageFarmBroker`` with the given tub maker and an empty
client configuration.
"""
if tub_maker is None:
tub_maker = lambda handler_overrides: new_tub()
return StorageFarmBroker(True, tub_maker, EMPTY_CLIENT_CONFIG)
@implementer(IStreamClientEndpoint)
@attr.s
class SpyEndpoint(object):
"""
Observe and record connection attempts.
:ivar list _append: A callable that accepts two-tuples. For each
attempted connection, it will be called with ``Deferred`` that was
returned and the ``Factory`` that was passed in.
"""
_append = attr.ib()
def connect(self, factory):
"""
Record the connection attempt.
:return: A ``Deferred`` that ``SpyEndpoint`` will not fire.
"""
d = Deferred()
self._append((d, factory))
return d
@implementer(IConnectionHintHandler)
@attr.s
class SpyHandler(object):
"""
A Foolscap connection hint handler for the "spy" hint type. Connections
are handled by just observing and recording them.
:ivar list _connects: A list containing one element for each connection
attempted with this handler. Each element is a two-tuple of the
``Deferred`` that was returned from ``connect`` and the factory that
was passed to ``connect``.
"""
_connects = attr.ib(default=attr.Factory(list))
def hint_to_endpoint(self, hint, reactor, update_status):
return (SpyEndpoint(self._connects.append), hint)
class TestStorageFarmBroker(unittest.TestCase):
def test_static_servers(self):
@ -525,7 +594,7 @@ storage:
ann:
anonymous-storage-FURL: {furl}
permutation-seed-base32: aaaaaaaaaaaaaaaaaaaaaaaa
""".format(furl=SOME_FURL.decode("utf-8"))
""".format(furl=SOME_FURL)
servers = yamlutil.safe_load(servers_yaml)
permseed = base32.a2b(b"aaaaaaaaaaaaaaaaaaaaaaaa")
broker.set_static_servers(servers["storage"])
@ -541,7 +610,7 @@ storage:
ann2 = {
"service-name": "storage",
"anonymous-storage-FURL": "pb://{}@nowhere/fake2".format(base32.b2a(b"1")),
"anonymous-storage-FURL": "pb://{}@nowhere/fake2".format(str(base32.b2a(b"1"), "utf-8")),
"permutation-seed-base32": "bbbbbbbbbbbbbbbbbbbbbbbb",
}
broker._got_announcement(key_s, ann2)
@ -585,18 +654,38 @@ storage:
@inlineCallbacks
def test_threshold_reached(self):
introducer = Mock()
"""
``StorageFarmBroker.when_connected_enough`` returns a ``Deferred`` which
only fires after the ``StorageFarmBroker`` has established at least as
many connections as requested.
"""
introducer = MemoryIntroducerClient(
new_tub(),
SOME_FURL,
b"",
None,
None,
None,
None,
)
new_tubs = []
def make_tub(*args, **kwargs):
return new_tubs.pop()
broker = make_broker(make_tub)
# Start the broker so that it will start Tubs attached to it so they
# will attempt to make connections as necessary so that we can observe
# those connections.
broker.startService()
self.addCleanup(broker.stopService)
done = broker.when_connected_enough(5)
broker.use_introducer(introducer)
# subscribes to "storage" to learn of new storage nodes
subscribe = introducer.mock_calls[0]
self.assertEqual(subscribe[0], 'subscribe_to')
self.assertEqual(subscribe[1][0], 'storage')
got_announcement = subscribe[1][1]
[subscribe] = introducer.subscribed_to
self.assertEqual(
subscribe.service_name,
"storage",
)
got_announcement = subscribe.cb
data = {
"service-name": "storage",
@ -605,15 +694,25 @@ storage:
}
def add_one_server(x):
data["anonymous-storage-FURL"] = b"pb://%s@nowhere/fake" % (base32.b2a(b"%d" % x),)
tub = Mock()
data["anonymous-storage-FURL"] = "pb://%s@spy:nowhere/fake" % (str(base32.b2a(b"%d" % x), "ascii"),)
tub = new_tub()
connects = []
spy = SpyHandler(connects)
tub.addConnectionHintHandler("spy", spy)
new_tubs.append(tub)
got_announcement(b'v0-1234-%d' % x, data)
self.assertEqual(tub.mock_calls[-1][0], 'connectTo')
got_connection = tub.mock_calls[-1][1][1]
rref = Mock()
rref.callRemote = Mock(return_value=succeed(1234))
got_connection(rref)
self.assertEqual(
1, len(connects),
"Expected one connection attempt, got {!r} instead".format(connects),
)
# Skip over all the Foolscap negotiation. It's complex with lots
# of pieces and I don't want to figure out how to fake
# it. -exarkun
native = broker.servers[b"v0-1234-%d" % (x,)]
rref = LocalWrapper(StorageServer(self.mktemp(), b"x" * 20))
native._got_connection(rref)
# first 4 shouldn't trigger connected_threashold
for x in range(4):

2
src/allmydata/test/test_upload.py
View File

@ -239,7 +239,7 @@ class FakeClient(object):
node_config=EMPTY_CLIENT_CONFIG,
)
for (serverid, rref) in servers:
ann = {"anonymous-storage-FURL": b"pb://%s@nowhere/fake" % base32.b2a(serverid),
ann = {"anonymous-storage-FURL": "pb://%s@nowhere/fake" % str(base32.b2a(serverid), "ascii"),
"permutation-seed-base32": base32.b2a(serverid) }
self.storage_broker.test_add_rref(serverid, rref, ann)
self.last_servers = [s[1] for s in servers]

3
src/allmydata/util/_python3.py
View File

@ -51,7 +51,10 @@ PORTED_MODULES = [
"allmydata.immutable.offloaded",
"allmydata.immutable.upload",
"allmydata.interfaces",
"allmydata.introducer.client",
"allmydata.introducer.common",
"allmydata.introducer.interfaces",
"allmydata.introducer.server",
"allmydata.monitor",
"allmydata.mutable.checker",
"allmydata.mutable.common",