ExternalVendorCode/tahoe-lafs
1
0
Fork 0
mirror of https://github.com/tahoe-lafs/tahoe-lafs.git synced 2025-01-18 10:46:24 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/master' into 3472.windows-coverage-reports

Browse Source
This commit is contained in:
Jean-Paul Calderone 2020-10-16 06:53:10 -04:00
commit d66ef8f506
17 changed files with 134 additions and 68 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

0
newsfragments/3454.minor Normal file
View File

0
newsfragments/3467.minor Normal file
View File

0
newsfragments/3470.minor Normal file
View File

0
newsfragments/3471.minor Normal file
View File

0
newsfragments/3473.minor Normal file
View File

12
src/allmydata/mutable/filenode.py
View File

@ -147,9 +147,9 @@ class MutableFileNode(object):
def _get_initial_contents(self, contents):
if contents is None:
return MutableData("")
return MutableData(b"")
if isinstance(contents, str):
if isinstance(contents, bytes):
return MutableData(contents)
if IMutableUploadable.providedBy(contents):
@ -884,9 +884,9 @@ class MutableFileVersion(object):
d = self._try_to_download_data()
def _apply(old_contents):
new_contents = modifier(old_contents, self._servermap, first_time)
precondition((isinstance(new_contents, str) or
precondition((isinstance(new_contents, bytes) or
new_contents is None),
"Modifier function must return a string "
"Modifier function must return bytes "
"or None")
if new_contents is None or new_contents == old_contents:
@ -960,7 +960,7 @@ class MutableFileVersion(object):
c = consumer.MemoryConsumer()
# modify will almost certainly write, so we need the privkey.
d = self._read(c, fetch_privkey=True)
d.addCallback(lambda mc: "".join(mc.chunks))
d.addCallback(lambda mc: b"".join(mc.chunks))
return d
@ -1076,7 +1076,7 @@ class MutableFileVersion(object):
start = offset
rest = offset + data.get_size()
new = old[:start]
new += "".join(data.read(data.get_size()))
new += b"".join(data.read(data.get_size()))
new += old[rest:]
return new
return self._modify(m, None)

48
src/allmydata/mutable/publish.py
View File

@ -1,5 +1,5 @@
import os, time
from six.moves import cStringIO as StringIO
from io import BytesIO
from itertools import count
from zope.interface import implementer
from twisted.internet import defer
@ -46,7 +46,7 @@ class PublishStatus(object):
self.size = None
self.status = "Not started"
self.progress = 0.0
self.counter = self.statusid_counter.next()
self.counter = next(self.statusid_counter)
self.started = time.time()
def add_per_server_time(self, server, elapsed):
@ -305,7 +305,7 @@ class Publish(object):
# Our update process fetched these for us. We need to update
# them in place as publishing happens.
self.blockhashes = {} # (shnum, [blochashes])
for (i, bht) in blockhashes.iteritems():
for (i, bht) in list(blockhashes.items()):
# We need to extract the leaves from our old hash tree.
old_segcount = mathutil.div_ceil(version[4],
version[3])
@ -313,7 +313,7 @@ class Publish(object):
bht = dict(enumerate(bht))
h.set_hashes(bht)
leaves = h[h.get_leaf_index(0):]
for j in xrange(self.num_segments - len(leaves)):
for j in range(self.num_segments - len(leaves)):
leaves.append(None)
assert len(leaves) >= self.num_segments
@ -509,10 +509,10 @@ class Publish(object):
# This will eventually hold the block hash chain for each share
# that we publish. We define it this way so that empty publishes
# will still have something to write to the remote slot.
self.blockhashes = dict([(i, []) for i in xrange(self.total_shares)])
for i in xrange(self.total_shares):
self.blockhashes = dict([(i, []) for i in range(self.total_shares)])
for i in range(self.total_shares):
blocks = self.blockhashes[i]
for j in xrange(self.num_segments):
for j in range(self.num_segments):
blocks.append(None)
self.sharehash_leaves = None # eventually [sharehashes]
self.sharehashes = {} # shnum -> [sharehash leaves necessary to
@ -526,7 +526,7 @@ class Publish(object):
return self.done_deferred
def _get_some_writer(self):
return list(self.writers.values()[0])[0]
return list(list(self.writers.values())[0])[0]
def _update_status(self):
self._status.set_status("Sending Shares: %d placed out of %d, "
@ -684,7 +684,7 @@ class Publish(object):
salt = os.urandom(16)
assert self._version == SDMF_VERSION
for shnum, writers in self.writers.iteritems():
for shnum, writers in self.writers.items():
for writer in writers:
writer.put_salt(salt)
@ -704,7 +704,7 @@ class Publish(object):
self.log("Pushing segment %d of %d" % (segnum + 1, self.num_segments))
data = self.data.read(segsize)
# XXX: This is dumb. Why return a list?
data = "".join(data)
data = b"".join(data)
assert len(data) == segsize, len(data)
@ -732,7 +732,7 @@ class Publish(object):
for i in range(len(crypttext_pieces)):
offset = i * piece_size
piece = crypttext[offset:offset+piece_size]
piece = piece + "\x00"*(piece_size - len(piece)) # padding
piece = piece + b"\x00"*(piece_size - len(piece)) # padding
crypttext_pieces[i] = piece
assert len(piece) == piece_size
d = fec.encode(crypttext_pieces)
@ -751,7 +751,7 @@ class Publish(object):
results, salt = encoded_and_salt
shares, shareids = results
self._status.set_status("Pushing segment")
for i in xrange(len(shares)):
for i in range(len(shares)):
sharedata = shares[i]
shareid = shareids[i]
if self._version == MDMF_VERSION:
@ -786,7 +786,7 @@ class Publish(object):
def push_encprivkey(self):
encprivkey = self._encprivkey
self._status.set_status("Pushing encrypted private key")
for shnum, writers in self.writers.iteritems():
for shnum, writers in self.writers.items():
for writer in writers:
writer.put_encprivkey(encprivkey)
@ -794,7 +794,7 @@ class Publish(object):
def push_blockhashes(self):
self.sharehash_leaves = [None] * len(self.blockhashes)
self._status.set_status("Building and pushing block hash tree")
for shnum, blockhashes in self.blockhashes.iteritems():
for shnum, blockhashes in list(self.blockhashes.items()):
t = hashtree.HashTree(blockhashes)
self.blockhashes[shnum] = list(t)
# set the leaf for future use.
@ -808,7 +808,7 @@ class Publish(object):
def push_sharehashes(self):
self._status.set_status("Building and pushing share hash chain")
share_hash_tree = hashtree.HashTree(self.sharehash_leaves)
for shnum in xrange(len(self.sharehash_leaves)):
for shnum in range(len(self.sharehash_leaves)):
needed_indices = share_hash_tree.needed_hashes(shnum)
self.sharehashes[shnum] = dict( [ (i, share_hash_tree[i])
for i in needed_indices] )
@ -824,7 +824,7 @@ class Publish(object):
# - Get the checkstring of the resulting layout; sign that.
# - Push the signature
self._status.set_status("Pushing root hashes and signature")
for shnum in xrange(self.total_shares):
for shnum in range(self.total_shares):
writers = self.writers[shnum]
for writer in writers:
writer.put_root_hash(self.root_hash)
@ -852,7 +852,7 @@ class Publish(object):
signable = self._get_some_writer().get_signable()
self.signature = rsa.sign_data(self._privkey, signable)
for (shnum, writers) in self.writers.iteritems():
for (shnum, writers) in self.writers.items():
for writer in writers:
writer.put_signature(self.signature)
self._status.timings['sign'] = time.time() - started
@ -867,7 +867,7 @@ class Publish(object):
ds = []
verification_key = rsa.der_string_from_verifying_key(self._pubkey)
for (shnum, writers) in self.writers.copy().iteritems():
for (shnum, writers) in list(self.writers.copy().items()):
for writer in writers:
writer.put_verification_key(verification_key)
self.num_outstanding += 1
@ -1003,7 +1003,7 @@ class Publish(object):
# TODO: Precompute this.
shares = []
for shnum, writers in self.writers.iteritems():
for shnum, writers in self.writers.items():
shares.extend([x.shnum for x in writers if x.server == server])
known_shnums = set(shares)
surprise_shares -= known_shnums
@ -1198,7 +1198,7 @@ class Publish(object):
class MutableFileHandle(object):
"""
I am a mutable uploadable built around a filehandle-like object,
usually either a StringIO instance or a handle to an actual file.
usually either a BytesIO instance or a handle to an actual file.
"""
def __init__(self, filehandle):
@ -1268,14 +1268,14 @@ class MutableFileHandle(object):
class MutableData(MutableFileHandle):
"""
I am a mutable uploadable built around a string, which I then cast
into a StringIO and treat as a filehandle.
into a BytesIO and treat as a filehandle.
"""
def __init__(self, s):
# Take a string and return a file-like uploadable.
assert isinstance(s, str)
assert isinstance(s, bytes)
MutableFileHandle.__init__(self, StringIO(s))
MutableFileHandle.__init__(self, BytesIO(s))
@implementer(IMutableUploadable)
@ -1361,7 +1361,7 @@ class TransformingUploadable(object):
self.log("reading %d bytes of new data" % length)
new_data = self._newdata.read(length)
new_data = "".join(new_data)
new_data = b"".join(new_data)
self._read_marker += len(old_start_data + new_data + old_end_data)

22
src/allmydata/test/mutable/test_datahandle.py
View File

@ -1,17 +1,29 @@
"""
Ported to Python 3.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
from future.utils import PY2
if PY2:
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
from twisted.trial import unittest
from allmydata.mutable.publish import MutableData
class DataHandle(unittest.TestCase):
def setUp(self):
self.test_data = "Test Data" * 50000
self.test_data = b"Test Data" * 50000
self.uploadable = MutableData(self.test_data)
def test_datahandle_read(self):
chunk_size = 10
for i in xrange(0, len(self.test_data), chunk_size):
for i in range(0, len(self.test_data), chunk_size):
data = self.uploadable.read(chunk_size)
data = "".join(data)
data = b"".join(data)
start = i
end = i + chunk_size
self.failUnlessEqual(data, self.test_data[start:end])
@ -28,7 +40,7 @@ class DataHandle(unittest.TestCase):
# disturbing the location of the seek pointer.
chunk_size = 100
data = self.uploadable.read(chunk_size)
self.failUnlessEqual("".join(data), self.test_data[:chunk_size])
self.failUnlessEqual(b"".join(data), self.test_data[:chunk_size])
# Now get the size.
size = self.uploadable.get_size()
@ -38,4 +50,4 @@ class DataHandle(unittest.TestCase):
more_data = self.uploadable.read(chunk_size)
start = chunk_size
end = chunk_size * 2
self.failUnlessEqual("".join(more_data), self.test_data[start:end])
self.failUnlessEqual(b"".join(more_data), self.test_data[start:end])

16
src/allmydata/test/mutable/test_different_encoding.py
View File

@ -1,3 +1,15 @@
"""
Ported to Python 3.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
from future.utils import PY2
if PY2:
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
from twisted.trial import unittest
from .util import FakeStorage, make_nodemaker
@ -10,7 +22,7 @@ class DifferentEncoding(unittest.TestCase):
# create a file with 3-of-20, then modify it with a client configured
# to do 3-of-10. #1510 tracks a failure here
self.nodemaker.default_encoding_parameters["n"] = 20
d = self.nodemaker.create_mutable_file("old contents")
d = self.nodemaker.create_mutable_file(b"old contents")
def _created(n):
filecap = n.get_cap().to_string()
del n # we want a new object, not the cached one
@ -19,6 +31,6 @@ class DifferentEncoding(unittest.TestCase):
return n2
d.addCallback(_created)
def modifier(old_contents, servermap, first_time):
return "new contents"
return b"new contents"
d.addCallback(lambda n: n.modify(modifier))
return d

34
src/allmydata/test/mutable/test_filehandle.py
View File

@ -1,21 +1,33 @@
"""
Ported to Python 3.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
from future.utils import PY2
if PY2:
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
import os
from six.moves import cStringIO as StringIO
from io import BytesIO
from twisted.trial import unittest
from allmydata.mutable.publish import MutableFileHandle
class FileHandle(unittest.TestCase):
def setUp(self):
self.test_data = "Test Data" * 50000
self.sio = StringIO(self.test_data)
self.test_data = b"Test Data" * 50000
self.sio = BytesIO(self.test_data)
self.uploadable = MutableFileHandle(self.sio)
def test_filehandle_read(self):
self.basedir = "mutable/FileHandle/test_filehandle_read"
chunk_size = 10
for i in xrange(0, len(self.test_data), chunk_size):
for i in range(0, len(self.test_data), chunk_size):
data = self.uploadable.read(chunk_size)
data = "".join(data)
data = b"".join(data)
start = i
end = i + chunk_size
self.failUnlessEqual(data, self.test_data[start:end])
@ -33,7 +45,7 @@ class FileHandle(unittest.TestCase):
# disturbing the location of the seek pointer.
chunk_size = 100
data = self.uploadable.read(chunk_size)
self.failUnlessEqual("".join(data), self.test_data[:chunk_size])
self.failUnlessEqual(b"".join(data), self.test_data[:chunk_size])
# Now get the size.
size = self.uploadable.get_size()
@ -43,26 +55,26 @@ class FileHandle(unittest.TestCase):
more_data = self.uploadable.read(chunk_size)
start = chunk_size
end = chunk_size * 2
self.failUnlessEqual("".join(more_data), self.test_data[start:end])
self.failUnlessEqual(b"".join(more_data), self.test_data[start:end])
def test_filehandle_file(self):
# Make sure that the MutableFileHandle works on a file as well
# as a StringIO object, since in some cases it will be asked to
# as a BytesIO object, since in some cases it will be asked to
# deal with files.
self.basedir = self.mktemp()
# necessary? What am I doing wrong here?
os.mkdir(self.basedir)
f_path = os.path.join(self.basedir, "test_file")
f = open(f_path, "w")
f = open(f_path, "wb")
f.write(self.test_data)
f.close()
f = open(f_path, "r")
f = open(f_path, "rb")
uploadable = MutableFileHandle(f)
data = uploadable.read(len(self.test_data))
self.failUnlessEqual("".join(data), self.test_data)
self.failUnlessEqual(b"".join(data), self.test_data)
size = uploadable.get_size()
self.failUnlessEqual(size, len(self.test_data))

16
src/allmydata/test/mutable/util.py
View File

@ -1,4 +1,6 @@
from six.moves import cStringIO as StringIO
from past.builtins import long
from io import BytesIO
import attr
from twisted.internet import defer, reactor
from foolscap.api import eventually, fireEventually
@ -75,8 +77,8 @@ class FakeStorage(object):
if peerid not in self._peers:
self._peers[peerid] = {}
shares = self._peers[peerid]
f = StringIO()
f.write(shares.get(shnum, ""))
f = BytesIO()
f.write(shares.get(shnum, b""))
f.seek(offset)
f.write(data)
shares[shnum] = f.getvalue()
@ -129,7 +131,7 @@ class FakeStorageServer(object):
readv = {}
for shnum, (testv, writev, new_length) in tw_vectors.items():
for (offset, length, op, specimen) in testv:
assert op in ("le", "eq", "ge")
assert op in (b"le", b"eq", b"ge")
# TODO: this isn't right, the read is controlled by read_vector,
# not by testv
readv[shnum] = [ specimen
@ -222,10 +224,10 @@ def make_peer(s, i):
:rtype: ``Peer``
"""
peerid = base32.b2a(tagged_hash("peerid", "%d" % i)[:20])
peerid = base32.b2a(tagged_hash(b"peerid", b"%d" % i)[:20])
fss = FakeStorageServer(peerid, s)
ann = {
"anonymous-storage-FURL": "pb://%s@nowhere/fake" % (peerid,),
"anonymous-storage-FURL": b"pb://%s@nowhere/fake" % (peerid,),
"permutation-seed-base32": peerid,
}
return Peer(peerid=peerid, storage_server=fss, announcement=ann)
@ -297,7 +299,7 @@ def make_nodemaker_with_storage_broker(storage_broker, keysize):
:param StorageFarmBroker peers: The storage broker to use.
"""
sh = client.SecretHolder("lease secret", "convergence secret")
sh = client.SecretHolder(b"lease secret", b"convergence secret")
keygen = client.KeyGenerator()
if keysize:
keygen.set_default_keysize(keysize)

37
src/allmydata/test/no_network.py
View File

@ -26,13 +26,19 @@ if PY2:
from past.builtins import unicode
import os
from base64 import b32encode
from functools import (
partial,
)
from zope.interface import implementer
from twisted.application import service
from twisted.internet import defer
from twisted.python.failure import Failure
from twisted.web.error import Error
from foolscap.api import Referenceable, fireEventually, RemoteException
from base64 import b32encode
from foolscap.ipb import (
IRemoteReference,
)
import treq
from allmydata.util.assertutil import _assert
@ -59,14 +65,29 @@ class IntentionalError(Exception):
class Marker(object):
pass
fireNow = partial(defer.succeed, None)
@implementer(IRemoteReference)
class LocalWrapper(object):
def __init__(self, original):
"""
A ``LocalWrapper`` presents the remote reference interface to a local
object which implements a ``RemoteInterface``.
"""
def __init__(self, original, fireEventually=fireEventually):
"""
:param Callable[[], Deferred[None]] fireEventually: Get a Deferred
that will fire at some point. This is used to control when
``callRemote`` calls the remote method. The default value allows
the reactor to iterate before the call happens. Use ``fireNow``
to call the remote method synchronously.
"""
self.original = original
self.broken = False
self.hung_until = None
self.post_call_notifier = None
self.disconnectors = {}
self.counter_by_methname = {}
self._fireEventually = fireEventually
def _clear_counters(self):
self.counter_by_methname = {}
@ -82,7 +103,7 @@ class LocalWrapper(object):
# selected return values.
def wrap(a):
if isinstance(a, Referenceable):
return LocalWrapper(a)
return self._wrap(a)
else:
return a
args = tuple([wrap(a) for a in args])
@ -110,7 +131,7 @@ class LocalWrapper(object):
return d2
return _really_call()
d = fireEventually()
d = self._fireEventually()
d.addCallback(lambda res: _call())
def _wrap_exception(f):
return Failure(RemoteException(f))
@ -124,10 +145,10 @@ class LocalWrapper(object):
if methname == "allocate_buckets":
(alreadygot, allocated) = res
for shnum in allocated:
allocated[shnum] = LocalWrapper(allocated[shnum])
allocated[shnum] = self._wrap(allocated[shnum])
if methname == "get_buckets":
for shnum in res:
res[shnum] = LocalWrapper(res[shnum])
res[shnum] = self._wrap(res[shnum])
return res
d.addCallback(_return_membrane)
if self.post_call_notifier:
@ -141,6 +162,10 @@ class LocalWrapper(object):
def dontNotifyOnDisconnect(self, marker):
del self.disconnectors[marker]
def _wrap(self, value):
return LocalWrapper(value, self._fireEventually)
def wrap_storage_server(original):
# Much of the upload/download code uses rref.version (which normally
# comes from rrefutil.add_version_to_remote_reference). To avoid using a

9
src/allmydata/test/test_python3.py
View File

@ -8,7 +8,7 @@ from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from future.utils import PY2
from future.utils import PY2, native_str
if PY2:
from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
@ -44,10 +44,9 @@ class Python3PortingEffortTests(SynchronousTestCase):
),
),
)
if PY2:
test_finished_porting.skip = "For some reason todo isn't working on Python 2 now"
else:
test_finished_porting.todo = "https://tahoe-lafs.org/trac/tahoe-lafs/milestone/Support%20Python%203 should be completed"
test_finished_porting.todo = native_str(
"https://tahoe-lafs.org/trac/tahoe-lafs/milestone/Support%20Python%203 should be completed",
)
def test_ported_modules_exist(self):
"""

2
src/allmydata/test/test_uri.py
View File

@ -11,7 +11,7 @@ from __future__ import unicode_literals
from future.utils import PY2
if PY2:
from future.builtins import filter, map, zip, ascii, chr, dict, hex, input, next, oct, open, pow, round, super, bytes, int, list, object, range, str, max, min # noqa: F401
from future.builtins import filter, map, zip, ascii, chr, dict, hex, input, next, oct, open, pow, round, super, bytes, list, object, range, str, max, min # noqa: F401
import os
from twisted.trial import unittest

1
src/allmydata/test/test_util.py
View File

@ -9,6 +9,7 @@ from __future__ import unicode_literals
from future.utils import PY2
if PY2:
# open is not here because we want to use native strings on Py2
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
import six
import os, time, sys

3
src/allmydata/util/_python3.py
View File

@ -89,6 +89,9 @@ PORTED_MODULES = [
]
PORTED_TEST_MODULES = [
"allmydata.test.mutable.test_datahandle",
"allmydata.test.mutable.test_different_encoding",
"allmydata.test.mutable.test_filehandle",
"allmydata.test.test_abbreviate",
"allmydata.test.test_base32",
"allmydata.test.test_base62",

2
src/allmydata/util/observer.py
View File

@ -61,7 +61,7 @@ class OneShotObserverList(object):
def _fire(self, result):
for w in self._watchers:
eventually(w.callback, result)
w.callback(result)
del self._watchers
self.__repr__ = self._fired_repr