ExternalVendorCode/tahoe-lafs
1
0
Fork 0
mirror of https://github.com/tahoe-lafs/tahoe-lafs.git synced 2025-02-20 09:46:18 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/master' into 3603.scripts

Browse Source
This commit is contained in:
Itamar Turner-Trauring 2021-03-15 10:50:13 -04:00
commit 3c8e18f0fc
14 changed files with 226 additions and 107 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

0
src/allmydata/test/test_import.py → newsfragments/3625.minor
View File

0
newsfragments/3628.minor Normal file
View File

1
newsfragments/3633.installation Normal file
View File

@ -0,0 +1 @@
Tahoe-LAFS now uses a forked version of txi2p (named txi2p-tahoe) with Python 3 support.

8
nix/tahoe-lafs.nix
View File

@ -24,11 +24,17 @@ python.pkgs.buildPythonPackage rec {
# tests with in a module.
# Many of these tests don't properly skip when i2p or tor dependencies are
# not supplied (and we are not supplying them).
# not supplied (and we are not supplying them). test_client.py fails because
# version is "unknown" on Nix.
# see https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3629 for the latter.
rm src/allmydata/test/test_i2p_provider.py
rm src/allmydata/test/test_connections.py
rm src/allmydata/test/cli/test_create.py
rm src/allmydata/test/test_client.py
# Since we're deleting files, this complains they're missing. For now Nix
# is Python 2-only, anyway, so these tests don't add anything yet.
rm src/allmydata/test/test_python3.py
'';

7
setup.py
View File

@ -151,8 +151,13 @@ tor_requires = [
]
i2p_requires = [
# txi2p has Python 3 support, but it's unreleased: https://github.com/str4d/txi2p/issues/10.
# txi2p has Python 3 support in master branch, but it has not been
# released -- see https://github.com/str4d/txi2p/issues/10. We
# could use a fork for Python 3 until txi2p's maintainers are back
# in action. For Python 2, we could continue using the txi2p
# version about which no one has complained to us so far.
"txi2p; python_version < '3.0'",
"txi2p-tahoe >= 0.3.5; python_version > '3.0'",
]
if len(sys.argv) > 1 and sys.argv[1] == '--fakedependency':

26
src/allmydata/client.py
View File

@ -1,4 +1,16 @@
from past.builtins import unicode
"""
Ported to Python 3.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
from future.utils import PY2
if PY2:
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, max, min # noqa: F401
# Don't use future str to prevent leaking future's newbytes into foolscap, which they break.
from past.builtins import unicode as str
import os, stat, time, weakref
from base64 import urlsafe_b64encode
@ -364,8 +376,8 @@ class _StoragePlugins(object):
"""
return set(
config.get_config(
"storage", "plugins", b""
).decode("ascii").split(u",")
"storage", "plugins", ""
).split(u",")
) - {u""}
@classmethod
@ -460,7 +472,7 @@ def create_introducer_clients(config, main_tub, _introducer_factory=None):
introducers = config.get_introducer_configuration()
for petname, (furl, cache_path) in introducers.items():
for petname, (furl, cache_path) in list(introducers.items()):
ic = _introducer_factory(
main_tub,
furl.encode("ascii"),
@ -679,7 +691,7 @@ class _Client(node.Node, pollmixin.PollMixin):
def init_secrets(self):
# configs are always unicode
def _unicode_make_secret():
return unicode(_make_secret(), "ascii")
return str(_make_secret(), "ascii")
lease_s = self.config.get_or_create_private_config(
"secret", _unicode_make_secret).encode("utf-8")
lease_secret = base32.a2b(lease_s)
@ -694,7 +706,7 @@ class _Client(node.Node, pollmixin.PollMixin):
def _make_key():
private_key, _ = ed25519.create_signing_keypair()
# Config values are always unicode:
return unicode(ed25519.string_from_signing_key(private_key) + b"\n", "utf-8")
return str(ed25519.string_from_signing_key(private_key) + b"\n", "utf-8")
private_key_str = self.config.get_or_create_private_config(
"node.privkey", _make_key).encode("utf-8")
@ -870,7 +882,7 @@ class _Client(node.Node, pollmixin.PollMixin):
"""
Register a storage server.
"""
config_key = b"storage-plugin.{}.furl".format(
config_key = "storage-plugin.{}.furl".format(
# Oops, why don't I have a better handle on this value?
announceable_storage_server.announcement[u"name"],
)

2
src/allmydata/storage_client.py
View File

@ -37,6 +37,7 @@ from __future__ import unicode_literals
from future.utils import PY2
if PY2:
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
from six import ensure_text
import re, time, hashlib
@ -198,6 +199,7 @@ class StorageFarmBroker(service.MultiService):
# doesn't really matter but it makes the logging behavior more
# predictable and easier to test (and at least one test does depend on
# this sorted order).
servers = {ensure_text(key): value for (key, value) in servers.items()}
for (server_id, server) in sorted(servers.items()):
try:
storage_server = self._make_storage_server(

2
src/allmydata/test/cli/common.py
View File

@ -52,6 +52,8 @@ class CLITestMixin(ReallyEqualMixin):
# Python functions want native strings. So ignore the requirements
# for passing arguments to another process and make sure this argument
# is a native string.
verb = ensure_str(verb)
args = [ensure_str(arg) for arg in args]
client_dir = ensure_str(self.get_clientdir(i=client_num))
nodeargs = [ b"--node-directory", client_dir ]
return run_cli(verb, *args, nodeargs=nodeargs, **kwargs)

2
src/allmydata/test/common.py
View File

@ -1059,7 +1059,7 @@ def _corrupt_mutable_share_data(data, debug=False):
assert prefix == MutableShareFile.MAGIC, "This function is designed to corrupt mutable shares of v1, and the magic number doesn't look right: %r vs %r" % (prefix, MutableShareFile.MAGIC)
data_offset = MutableShareFile.DATA_OFFSET
sharetype = data[data_offset:data_offset+1]
assert sharetype == "\x00", "non-SDMF mutable shares not supported"
assert sharetype == b"\x00", "non-SDMF mutable shares not supported"
(version, ig_seqnum, ig_roothash, ig_IV, ig_k, ig_N, ig_segsize,
ig_datalen, offsets) = unpack_header(data[data_offset:])
assert version == 0, "this function only handles v0 SDMF files"

12
src/allmydata/test/matchers.py
View File

@ -1,6 +1,16 @@
"""
Testtools-style matchers useful to the Tahoe-LAFS test suite.
Ported to Python 3.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
from future.utils import PY2
if PY2:
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
import attr
@ -51,7 +61,7 @@ class MatchesNodePublicKey(object):
:return Mismatch: If the keys don't match.
"""
config = read_config(self.basedir, u"tub.port")
privkey_bytes = config.get_private_config("node.privkey")
privkey_bytes = config.get_private_config("node.privkey").encode("utf-8")
private_key = ed25519.signing_keypair_from_string(privkey_bytes)[0]
signature = ed25519.sign_data(private_key, b"")
other_public_key = ed25519.verifying_key_from_signing_key(other)

112
src/allmydata/test/test_client.py
View File

@ -1,3 +1,15 @@
"""
Ported to Python 3.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
from future.utils import PY2
if PY2:
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
import os, sys
from functools import (
partial,
@ -21,7 +33,6 @@ from hypothesis.strategies import (
)
from eliot.testing import (
capture_logging,
assertHasAction,
)
from twisted.trial import unittest
@ -62,6 +73,7 @@ from allmydata.util import (
encodingutil,
configutil,
)
from allmydata.util.eliotutil import capture_logging
from allmydata.util.fileutil import abspath_expanduser_unicode
from allmydata.interfaces import IFilesystemNode, IFileNode, \
IImmutableFileNode, IMutableFileNode, IDirectoryNode
@ -186,7 +198,7 @@ class Basic(testutil.ReallyEqualMixin, unittest.TestCase):
basedir,
"client.port",
)
abs_basedir = fileutil.abspath_expanduser_unicode(unicode(basedir)).encode(sys.getfilesystemencoding())
abs_basedir = fileutil.abspath_expanduser_unicode(str(basedir))
self.failUnlessIn(os.path.join(abs_basedir, "introducer.furl"), e.args[0])
self.failUnlessIn(os.path.join(abs_basedir, "no_storage"), e.args[0])
self.failUnlessIn(os.path.join(abs_basedir, "readonly_storage"), e.args[0])
@ -234,7 +246,7 @@ class Basic(testutil.ReallyEqualMixin, unittest.TestCase):
fileutil.write(os.path.join(basedir, "tahoe.cfg"),
BASECONFIG)
c = yield client.create_client(basedir)
self.failUnless(c.get_long_nodeid().startswith("v0-"))
self.failUnless(c.get_long_nodeid().startswith(b"v0-"))
@defer.inlineCallbacks
def test_nodekey_no_storage(self):
@ -246,7 +258,7 @@ class Basic(testutil.ReallyEqualMixin, unittest.TestCase):
fileutil.write(os.path.join(basedir, "tahoe.cfg"),
BASECONFIG + "[storage]\n" + "enabled = false\n")
c = yield client.create_client(basedir)
self.failUnless(c.get_long_nodeid().startswith("v0-"))
self.failUnless(c.get_long_nodeid().startswith(b"v0-"))
def test_storage_anonymous_enabled_by_default(self):
"""
@ -431,6 +443,9 @@ class Basic(testutil.ReallyEqualMixin, unittest.TestCase):
"""
generic helper for following storage_dir tests
"""
assert isinstance(basedir, str)
assert isinstance(storage_path, (str, type(None)))
assert isinstance(expected_path, str)
os.mkdir(basedir)
cfg_path = os.path.join(basedir, "tahoe.cfg")
fileutil.write(
@ -477,7 +492,7 @@ class Basic(testutil.ReallyEqualMixin, unittest.TestCase):
the node's basedir.
"""
basedir = u"client.Basic.test_relative_storage_dir"
config_path = b"myowndir"
config_path = u"myowndir"
expected_path = os.path.join(
abspath_expanduser_unicode(basedir),
u"myowndir",
@ -504,7 +519,7 @@ class Basic(testutil.ReallyEqualMixin, unittest.TestCase):
expected_path = abspath_expanduser_unicode(
u"client.Basic.test_absolute_storage_dir_myowndir/" + base
)
config_path = expected_path.encode("utf-8")
config_path = expected_path
return self._storage_dir_test(
basedir,
config_path,
@ -515,33 +530,62 @@ class Basic(testutil.ReallyEqualMixin, unittest.TestCase):
return [ s.get_longname() for s in sb.get_servers_for_psi(key) ]
def test_permute(self):
"""
Permutations need to be stable across Tahoe releases, which is why we
hardcode a specific expected order.
This is because the order of these results determines which servers a
client will choose to place shares on and which servers it will consult
(and in what order) when trying to retrieve those shares. If the order
ever changes, all already-placed shares become (at best) harder to find
or (at worst) impossible to find.
"""
sb = StorageFarmBroker(True, None, EMPTY_CLIENT_CONFIG)
for k in ["%d" % i for i in range(5)]:
ks = [b"%d" % i for i in range(5)]
for k in ks:
ann = {"anonymous-storage-FURL": SOME_FURL,
"permutation-seed-base32": base32.b2a(k) }
sb.test_add_rref(k, "rref", ann)
self.failUnlessReallyEqual(self._permute(sb, "one"), ['3','1','0','4','2'])
self.failUnlessReallyEqual(self._permute(sb, "two"), ['0','4','2','1','3'])
one = self._permute(sb, b"one")
two = self._permute(sb, b"two")
self.failUnlessReallyEqual(one, [b'3',b'1',b'0',b'4',b'2'])
self.failUnlessReallyEqual(two, [b'0',b'4',b'2',b'1',b'3'])
self.assertEqual(sorted(one), ks)
self.assertEqual(sorted(two), ks)
self.assertNotEqual(one, two)
sb.servers.clear()
self.failUnlessReallyEqual(self._permute(sb, "one"), [])
self.failUnlessReallyEqual(self._permute(sb, b"one"), [])
def test_permute_with_preferred(self):
"""
Permutations need to be stable across Tahoe releases, which is why we
hardcode a specific expected order. In this case, two values are
preferred and should come first.
"""
sb = StorageFarmBroker(
True,
None,
EMPTY_CLIENT_CONFIG,
StorageClientConfig(preferred_peers=['1','4']),
StorageClientConfig(preferred_peers=[b'1',b'4']),
)
for k in ["%d" % i for i in range(5)]:
ks = [b"%d" % i for i in range(5)]
for k in [b"%d" % i for i in range(5)]:
ann = {"anonymous-storage-FURL": SOME_FURL,
"permutation-seed-base32": base32.b2a(k) }
sb.test_add_rref(k, "rref", ann)
self.failUnlessReallyEqual(self._permute(sb, "one"), ['1','4','3','0','2'])
self.failUnlessReallyEqual(self._permute(sb, "two"), ['4','1','0','2','3'])
one = self._permute(sb, b"one")
two = self._permute(sb, b"two")
self.failUnlessReallyEqual(b"".join(one), b'14302')
self.failUnlessReallyEqual(b"".join(two), b'41023')
self.assertEqual(sorted(one), ks)
self.assertEqual(sorted(one[:2]), [b"1", b"4"])
self.assertEqual(sorted(two), ks)
self.assertEqual(sorted(two[:2]), [b"1", b"4"])
self.assertNotEqual(one, two)
sb.servers.clear()
self.failUnlessReallyEqual(self._permute(sb, "one"), [])
self.failUnlessReallyEqual(self._permute(sb, b"one"), [])
@defer.inlineCallbacks
def test_versions(self):
@ -557,8 +601,8 @@ class Basic(testutil.ReallyEqualMixin, unittest.TestCase):
c = yield client.create_client(basedir)
ss = c.getServiceNamed("storage")
verdict = ss.remote_get_version()
self.failUnlessReallyEqual(verdict["application-version"],
str(allmydata.__full_version__))
self.failUnlessReallyEqual(verdict[b"application-version"],
allmydata.__full_version__.encode("ascii"))
self.failIfEqual(str(allmydata.__version__), "unknown")
self.failUnless("." in str(allmydata.__full_version__),
"non-numeric version in '%s'" % allmydata.__version__)
@ -783,7 +827,7 @@ class StaticServers(Fixture):
for (serverid, announcement)
in self._server_details
},
}))
}).encode("utf-8"))
class StorageClients(SyncTestCase):
@ -832,7 +876,7 @@ class StorageClients(SyncTestCase):
succeeded(
AfterPreprocessing(
get_known_server_details,
Equals([(serverid, announcement)]),
Equals([(serverid.encode("utf-8"), announcement)]),
),
),
)
@ -859,7 +903,7 @@ class StorageClients(SyncTestCase):
self.useFixture(
StaticServers(
self.basedir,
[(serverid, announcement),
[(serverid.encode("ascii"), announcement),
# Along with a "bad" server announcement. Order in this list
# doesn't matter, yaml serializer and Python dicts are going
# to shuffle everything around kind of randomly.
@ -876,7 +920,7 @@ class StorageClients(SyncTestCase):
AfterPreprocessing(
get_known_server_details,
# It should have the good server details.
Equals([(serverid, announcement)]),
Equals([(serverid.encode("utf-8"), announcement)]),
),
),
)
@ -903,7 +947,7 @@ class Run(unittest.TestCase, testutil.StallMixin):
private.makedirs()
dummy = "pb://wl74cyahejagspqgy4x5ukrvfnevlknt@127.0.0.1:58889/bogus"
write_introducer(basedir, "someintroducer", dummy)
basedir.child("tahoe.cfg").setContent(BASECONFIG)
basedir.child("tahoe.cfg").setContent(BASECONFIG.encode("ascii"))
basedir.child(client._Client.EXIT_TRIGGER_FILE).touch()
yield client.create_client(basedir.path)
@ -914,7 +958,7 @@ class Run(unittest.TestCase, testutil.StallMixin):
private.makedirs()
dummy = "pb://wl74cyahejagspqgy4x5ukrvfnevlknt@127.0.0.1:58889/bogus"
write_introducer(basedir, "someintroducer", dummy)
basedir.child("tahoe.cfg").setContent(BASECONFIG)
basedir.child("tahoe.cfg").setContent(BASECONFIG. encode("ascii"))
c1 = yield client.create_client(basedir.path)
c1.setServiceParent(self.sparent)
@ -1041,7 +1085,7 @@ class NodeMakerTests(testutil.ReallyEqualMixin, AsyncBrokenTestCase):
fileutil.write(os.path.join(basedir, "tahoe.cfg"), BASECONFIG)
c = yield client.create_client(basedir)
n = c.create_node_from_uri("URI:CHK:6nmrpsubgbe57udnexlkiwzmlu:bjt7j6hshrlmadjyr7otq3dc24end5meo5xcr5xe5r663po6itmq:3:10:7277")
n = c.create_node_from_uri(b"URI:CHK:6nmrpsubgbe57udnexlkiwzmlu:bjt7j6hshrlmadjyr7otq3dc24end5meo5xcr5xe5r663po6itmq:3:10:7277")
self.failUnless(IFilesystemNode.providedBy(n))
self.failUnless(IFileNode.providedBy(n))
self.failUnless(IImmutableFileNode.providedBy(n))
@ -1059,10 +1103,10 @@ class NodeMakerTests(testutil.ReallyEqualMixin, AsyncBrokenTestCase):
# current fix for this (hopefully to be superceded by a better fix
# eventually) is to prevent re-use of filenodes, so the NodeMaker is
# hereby required *not* to cache and re-use filenodes for CHKs.
other_n = c.create_node_from_uri("URI:CHK:6nmrpsubgbe57udnexlkiwzmlu:bjt7j6hshrlmadjyr7otq3dc24end5meo5xcr5xe5r663po6itmq:3:10:7277")
other_n = c.create_node_from_uri(b"URI:CHK:6nmrpsubgbe57udnexlkiwzmlu:bjt7j6hshrlmadjyr7otq3dc24end5meo5xcr5xe5r663po6itmq:3:10:7277")
self.failIf(n is other_n, (n, other_n))
n = c.create_node_from_uri("URI:LIT:n5xgk")
n = c.create_node_from_uri(b"URI:LIT:n5xgk")
self.failUnless(IFilesystemNode.providedBy(n))
self.failUnless(IFileNode.providedBy(n))
self.failUnless(IImmutableFileNode.providedBy(n))
@ -1071,7 +1115,7 @@ class NodeMakerTests(testutil.ReallyEqualMixin, AsyncBrokenTestCase):
self.failUnless(n.is_readonly())
self.failIf(n.is_mutable())
n = c.create_node_from_uri("URI:SSK:n6x24zd3seu725yluj75q5boaa:mm6yoqjhl6ueh7iereldqxue4nene4wl7rqfjfybqrehdqmqskvq")
n = c.create_node_from_uri(b"URI:SSK:n6x24zd3seu725yluj75q5boaa:mm6yoqjhl6ueh7iereldqxue4nene4wl7rqfjfybqrehdqmqskvq")
self.failUnless(IFilesystemNode.providedBy(n))
self.failUnless(IFileNode.providedBy(n))
self.failIf(IImmutableFileNode.providedBy(n))
@ -1080,7 +1124,7 @@ class NodeMakerTests(testutil.ReallyEqualMixin, AsyncBrokenTestCase):
self.failIf(n.is_readonly())
self.failUnless(n.is_mutable())
n = c.create_node_from_uri("URI:SSK-RO:b7sr5qsifnicca7cbk3rhrhbvq:mm6yoqjhl6ueh7iereldqxue4nene4wl7rqfjfybqrehdqmqskvq")
n = c.create_node_from_uri(b"URI:SSK-RO:b7sr5qsifnicca7cbk3rhrhbvq:mm6yoqjhl6ueh7iereldqxue4nene4wl7rqfjfybqrehdqmqskvq")
self.failUnless(IFilesystemNode.providedBy(n))
self.failUnless(IFileNode.providedBy(n))
self.failIf(IImmutableFileNode.providedBy(n))
@ -1089,7 +1133,7 @@ class NodeMakerTests(testutil.ReallyEqualMixin, AsyncBrokenTestCase):
self.failUnless(n.is_readonly())
self.failUnless(n.is_mutable())
n = c.create_node_from_uri("URI:DIR2:n6x24zd3seu725yluj75q5boaa:mm6yoqjhl6ueh7iereldqxue4nene4wl7rqfjfybqrehdqmqskvq")
n = c.create_node_from_uri(b"URI:DIR2:n6x24zd3seu725yluj75q5boaa:mm6yoqjhl6ueh7iereldqxue4nene4wl7rqfjfybqrehdqmqskvq")
self.failUnless(IFilesystemNode.providedBy(n))
self.failIf(IFileNode.providedBy(n))
self.failIf(IImmutableFileNode.providedBy(n))
@ -1098,7 +1142,7 @@ class NodeMakerTests(testutil.ReallyEqualMixin, AsyncBrokenTestCase):
self.failIf(n.is_readonly())
self.failUnless(n.is_mutable())
n = c.create_node_from_uri("URI:DIR2-RO:b7sr5qsifnicca7cbk3rhrhbvq:mm6yoqjhl6ueh7iereldqxue4nene4wl7rqfjfybqrehdqmqskvq")
n = c.create_node_from_uri(b"URI:DIR2-RO:b7sr5qsifnicca7cbk3rhrhbvq:mm6yoqjhl6ueh7iereldqxue4nene4wl7rqfjfybqrehdqmqskvq")
self.failUnless(IFilesystemNode.providedBy(n))
self.failIf(IFileNode.providedBy(n))
self.failIf(IImmutableFileNode.providedBy(n))
@ -1107,8 +1151,8 @@ class NodeMakerTests(testutil.ReallyEqualMixin, AsyncBrokenTestCase):
self.failUnless(n.is_readonly())
self.failUnless(n.is_mutable())
unknown_rw = "lafs://from_the_future"
unknown_ro = "lafs://readonly_from_the_future"
unknown_rw = b"lafs://from_the_future"
unknown_ro = b"lafs://readonly_from_the_future"
n = c.create_node_from_uri(unknown_rw, unknown_ro)
self.failUnless(IFilesystemNode.providedBy(n))
self.failIf(IFileNode.providedBy(n))
@ -1118,7 +1162,7 @@ class NodeMakerTests(testutil.ReallyEqualMixin, AsyncBrokenTestCase):
self.failUnless(n.is_unknown())
self.failUnlessReallyEqual(n.get_uri(), unknown_rw)
self.failUnlessReallyEqual(n.get_write_uri(), unknown_rw)
self.failUnlessReallyEqual(n.get_readonly_uri(), "ro." + unknown_ro)
self.failUnlessReallyEqual(n.get_readonly_uri(), b"ro." + unknown_ro)
# Note: it isn't that we *intend* to deploy non-ASCII caps in
# the future, it is that we want to make sure older Tahoe-LAFS
@ -1135,7 +1179,7 @@ class NodeMakerTests(testutil.ReallyEqualMixin, AsyncBrokenTestCase):
self.failUnless(n.is_unknown())
self.failUnlessReallyEqual(n.get_uri(), unknown_rw)
self.failUnlessReallyEqual(n.get_write_uri(), unknown_rw)
self.failUnlessReallyEqual(n.get_readonly_uri(), "ro." + unknown_ro)
self.failUnlessReallyEqual(n.get_readonly_uri(), b"ro." + unknown_ro)

142
src/allmydata/test/test_deepcheck.py
View File

@ -1,4 +1,10 @@
import os, json, urllib
"""
Ported to Python 3.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
# Python 2 compatibility
# Can't use `builtins.str` because something deep in Twisted callbacks ends up repr'ing
@ -11,7 +17,13 @@ import os, json, urllib
# (Pdb) pp data
# '334:12:b\'mutable-good\',90:URI:SSK-RO:...
from past.builtins import unicode as str
from future.utils import native_str
from future.utils import PY3, PY2
if PY2:
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, max, min # noqa: F401
import os, json
from urllib.parse import quote as url_quote
from twisted.trial import unittest
from twisted.internet import defer
@ -29,31 +41,37 @@ from allmydata.uri import LiteralFileURI
from allmydata.test.common import ErrorMixin, _corrupt_mutable_share_data, \
ShouldFailMixin
from .common_util import StallMixin, run_cli
from .common_util import StallMixin, run_cli_unicode
from .common_web import do_http
from allmydata.test.no_network import GridTestMixin
from .cli.common import CLITestMixin
def run_cli(verb, *argv):
"""Match usage in existing tests by accept *args."""
return run_cli_unicode(verb, list(argv))
class MutableChecker(GridTestMixin, unittest.TestCase, ErrorMixin):
def test_good(self):
self.basedir = "deepcheck/MutableChecker/good"
self.set_up_grid()
CONTENTS = "a little bit of data"
CONTENTS = b"a little bit of data"
CONTENTS_uploadable = MutableData(CONTENTS)
d = self.g.clients[0].create_mutable_file(CONTENTS_uploadable)
def _created(node):
self.node = node
self.fileurl = "uri/" + urllib.quote(node.get_uri())
self.fileurl = "uri/" + url_quote(node.get_uri())
d.addCallback(_created)
# now make sure the webapi verifier sees no problems
d.addCallback(lambda ign: self.GET(self.fileurl+"?t=check&verify=true",
method="POST"))
def _got_results(out):
self.failUnless("<span>Healthy : Healthy</span>" in out, out)
self.failUnless("Recoverable Versions: 10*seq1-" in out, out)
self.failIf("Not Healthy!" in out, out)
self.failIf("Unhealthy" in out, out)
self.failIf("Corrupt Shares" in out, out)
self.failUnless(b"<span>Healthy : Healthy</span>" in out, out)
self.failUnless(b"Recoverable Versions: 10*seq1-" in out, out)
self.failIf(b"Not Healthy!" in out, out)
self.failIf(b"Unhealthy" in out, out)
self.failIf(b"Corrupt Shares" in out, out)
d.addCallback(_got_results)
d.addErrback(self.explain_web_error)
return d
@ -61,12 +79,12 @@ class MutableChecker(GridTestMixin, unittest.TestCase, ErrorMixin):
def test_corrupt(self):
self.basedir = "deepcheck/MutableChecker/corrupt"
self.set_up_grid()
CONTENTS = "a little bit of data"
CONTENTS = b"a little bit of data"
CONTENTS_uploadable = MutableData(CONTENTS)
d = self.g.clients[0].create_mutable_file(CONTENTS_uploadable)
def _stash_and_corrupt(node):
self.node = node
self.fileurl = "uri/" + urllib.quote(node.get_uri())
self.fileurl = "uri/" + url_quote(node.get_uri())
self.corrupt_shares_numbered(node.get_uri(), [0],
_corrupt_mutable_share_data)
d.addCallback(_stash_and_corrupt)
@ -74,9 +92,9 @@ class MutableChecker(GridTestMixin, unittest.TestCase, ErrorMixin):
d.addCallback(lambda ign: self.GET(self.fileurl+"?t=check&verify=true",
method="POST"))
def _got_results(out):
self.failUnless("Not Healthy!" in out, out)
self.failUnless("Unhealthy: best version has only 9 shares (encoding is 3-of-10)" in out, out)
self.failUnless("Corrupt Shares:" in out, out)
self.failUnless(b"Not Healthy!" in out, out)
self.failUnless(b"Unhealthy: best version has only 9 shares (encoding is 3-of-10)" in out, out)
self.failUnless(b"Corrupt Shares:" in out, out)
d.addCallback(_got_results)
# now make sure the webapi repairer can fix it
@ -84,13 +102,13 @@ class MutableChecker(GridTestMixin, unittest.TestCase, ErrorMixin):
self.GET(self.fileurl+"?t=check&verify=true&repair=true",
method="POST"))
def _got_repair_results(out):
self.failUnless("<div>Repair successful</div>" in out, out)
self.failUnless(b"<div>Repair successful</div>" in out, out)
d.addCallback(_got_repair_results)
d.addCallback(lambda ign: self.GET(self.fileurl+"?t=check&verify=true",
method="POST"))
def _got_postrepair_results(out):
self.failIf("Not Healthy!" in out, out)
self.failUnless("Recoverable Versions: 10*seq" in out, out)
self.failIf(b"Not Healthy!" in out, out)
self.failUnless(b"Recoverable Versions: 10*seq" in out, out)
d.addCallback(_got_postrepair_results)
d.addErrback(self.explain_web_error)
@ -99,21 +117,21 @@ class MutableChecker(GridTestMixin, unittest.TestCase, ErrorMixin):
def test_delete_share(self):
self.basedir = "deepcheck/MutableChecker/delete_share"
self.set_up_grid()
CONTENTS = "a little bit of data"
CONTENTS = b"a little bit of data"
CONTENTS_uploadable = MutableData(CONTENTS)
d = self.g.clients[0].create_mutable_file(CONTENTS_uploadable)
def _stash_and_delete(node):
self.node = node
self.fileurl = "uri/" + urllib.quote(node.get_uri())
self.fileurl = "uri/" + url_quote(node.get_uri())
self.delete_shares_numbered(node.get_uri(), [0])
d.addCallback(_stash_and_delete)
# now make sure the webapi checker notices it
d.addCallback(lambda ign: self.GET(self.fileurl+"?t=check&verify=false",
method="POST"))
def _got_results(out):
self.failUnless("Not Healthy!" in out, out)
self.failUnless("Unhealthy: best version has only 9 shares (encoding is 3-of-10)" in out, out)
self.failIf("Corrupt Shares" in out, out)
self.failUnless(b"Not Healthy!" in out, out)
self.failUnless(b"Unhealthy: best version has only 9 shares (encoding is 3-of-10)" in out, out)
self.failIf(b"Corrupt Shares" in out, out)
d.addCallback(_got_results)
# now make sure the webapi repairer can fix it
@ -121,13 +139,13 @@ class MutableChecker(GridTestMixin, unittest.TestCase, ErrorMixin):
self.GET(self.fileurl+"?t=check&verify=false&repair=true",
method="POST"))
def _got_repair_results(out):
self.failUnless("Repair successful" in out)
self.failUnless(b"Repair successful" in out)
d.addCallback(_got_repair_results)
d.addCallback(lambda ign: self.GET(self.fileurl+"?t=check&verify=false",
method="POST"))
def _got_postrepair_results(out):
self.failIf("Not Healthy!" in out, out)
self.failUnless("Recoverable Versions: 10*seq" in out)
self.failIf(b"Not Healthy!" in out, out)
self.failUnless(b"Recoverable Versions: 10*seq" in out)
d.addCallback(_got_postrepair_results)
d.addErrback(self.explain_web_error)
@ -152,7 +170,7 @@ class DeepCheckBase(GridTestMixin, ErrorMixin, StallMixin, ShouldFailMixin,
return data
def parse_streamed_json(self, s):
for unit in s.split("\n"):
for unit in s.split(b"\n"):
if not unit:
# stream should end with a newline, so split returns ""
continue
@ -165,14 +183,14 @@ class DeepCheckBase(GridTestMixin, ErrorMixin, StallMixin, ShouldFailMixin,
@inlineCallbacks
def web(self, n, method="GET", **kwargs):
# returns (data, url)
url = (self.client_baseurls[0] + "uri/%s" % urllib.quote(n.get_uri())
+ "?" + "&".join(["%s=%s" % (k,v) for (k,v) in kwargs.items()]))
url = (self.client_baseurls[0] + "uri/%s" % url_quote(n.get_uri())
+ "?" + "&".join(["%s=%s" % (k,str(v, "ascii") if isinstance(v, bytes) else v) for (k,v) in kwargs.items()]))
data = yield do_http(method, url, browser_like_redirects=True)
returnValue((data,url))
@inlineCallbacks
def wait_for_operation(self, ophandle):
url = self.client_baseurls[0] + "operations/" + ophandle
url = self.client_baseurls[0] + "operations/" + str(ophandle, "ascii")
url += "?t=status&output=JSON"
while True:
body = yield do_http("get", url)
@ -184,7 +202,7 @@ class DeepCheckBase(GridTestMixin, ErrorMixin, StallMixin, ShouldFailMixin,
@inlineCallbacks
def get_operation_results(self, ophandle, output=None):
url = self.client_baseurls[0] + "operations/" + ophandle
url = self.client_baseurls[0] + "operations/" + str(ophandle, "ascii")
url += "?t=status"
if output:
url += "&output=" + output
@ -220,36 +238,36 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase):
self.root_uri = n.get_uri()
d.addCallback(_created_root)
d.addCallback(lambda ign:
c0.create_mutable_file(MutableData("mutable file contents")))
c0.create_mutable_file(MutableData(b"mutable file contents")))
d.addCallback(lambda n: self.root.set_node(u"mutable", n))
def _created_mutable(n):
self.mutable = n
self.mutable_uri = n.get_uri()
d.addCallback(_created_mutable)
large = upload.Data("Lots of data\n" * 1000, None)
large = upload.Data(b"Lots of data\n" * 1000, None)
d.addCallback(lambda ign: self.root.add_file(u"large", large))
def _created_large(n):
self.large = n
self.large_uri = n.get_uri()
d.addCallback(_created_large)
small = upload.Data("Small enough for a LIT", None)
small = upload.Data(b"Small enough for a LIT", None)
d.addCallback(lambda ign: self.root.add_file(u"small", small))
def _created_small(n):
self.small = n
self.small_uri = n.get_uri()
d.addCallback(_created_small)
small2 = upload.Data("Small enough for a LIT too", None)
small2 = upload.Data(b"Small enough for a LIT too", None)
d.addCallback(lambda ign: self.root.add_file(u"small2", small2))
def _created_small2(n):
self.small2 = n
self.small2_uri = n.get_uri()
d.addCallback(_created_small2)
empty_litdir_uri = "URI:DIR2-LIT:"
tiny_litdir_uri = "URI:DIR2-LIT:gqytunj2onug64tufqzdcosvkjetutcjkq5gw4tvm5vwszdgnz5hgyzufqydulbshj5x2lbm" # contains one child which is itself also LIT
empty_litdir_uri = b"URI:DIR2-LIT:"
tiny_litdir_uri = b"URI:DIR2-LIT:gqytunj2onug64tufqzdcosvkjetutcjkq5gw4tvm5vwszdgnz5hgyzufqydulbshj5x2lbm" # contains one child which is itself also LIT
d.addCallback(lambda ign: self.root._create_and_validate_node(None, empty_litdir_uri, name=u"test_deepcheck empty_lit_dir"))
def _created_empty_lit_dir(n):
@ -292,7 +310,7 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase):
sorted(self.g.get_all_serverids()),
where)
all_serverids = set()
for (shareid, servers) in cr.get_sharemap().items():
for (shareid, servers) in list(cr.get_sharemap().items()):
all_serverids.update([s.get_serverid() for s in servers])
self.failUnlessEqual(sorted(all_serverids),
sorted(self.g.get_all_serverids()),
@ -397,14 +415,14 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase):
mutable = [f for f in files
if f["cap"] is not None
and f["cap"].startswith("URI:SSK:")][0]
self.failUnlessEqual(mutable["cap"], self.mutable_uri)
self.failUnlessEqual(mutable["cap"].encode("ascii"), self.mutable_uri)
self.failIfEqual(mutable["cap"], mutable["verifycap"])
self.failUnlessEqual(mutable["cap"], mutable["repaircap"])
# for immutable file, verifycap==repaircap!=filecap
large = [f for f in files
if f["cap"] is not None
and f["cap"].startswith("URI:CHK:")][0]
self.failUnlessEqual(large["cap"], self.large_uri)
self.failUnlessEqual(large["cap"].encode("ascii"), self.large_uri)
self.failIfEqual(large["cap"], large["verifycap"])
self.failUnlessEqual(large["verifycap"], large["repaircap"])
self.check_stats_good(stats)
@ -524,7 +542,7 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase):
def json_check_is_healthy(self, data, n, where, incomplete=False):
self.failUnlessEqual(data["storage-index"],
base32.b2a(n.get_storage_index()), where)
str(base32.b2a(n.get_storage_index()), "ascii"), where)
self.failUnless("summary" in data, (where, data))
self.failUnlessEqual(data["summary"].lower(), "healthy",
"%s: '%s'" % (where, data["summary"]))
@ -550,7 +568,7 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase):
where)
self.failUnless("sharemap" in r, where)
all_serverids = set()
for (shareid, serverids_s) in r["sharemap"].items():
for (shareid, serverids_s) in list(r["sharemap"].items()):
all_serverids.update(serverids_s)
self.failUnlessEqual(sorted(all_serverids),
sorted([idlib.nodeid_b2a(sid)
@ -562,7 +580,7 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase):
def json_check_and_repair_is_healthy(self, data, n, where, incomplete=False):
self.failUnlessEqual(data["storage-index"],
base32.b2a(n.get_storage_index()), where)
str(base32.b2a(n.get_storage_index()), "ascii"), where)
self.failUnlessEqual(data["repair-attempted"], False, where)
self.json_check_is_healthy(data["pre-repair-results"],
n, where, incomplete)
@ -571,7 +589,7 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase):
def json_full_deepcheck_is_healthy(self, data, n, where):
self.failUnlessEqual(data["root-storage-index"],
base32.b2a(n.get_storage_index()), where)
str(base32.b2a(n.get_storage_index()), "ascii"), where)
self.failUnlessEqual(data["count-objects-checked"], 3, where)
self.failUnlessEqual(data["count-objects-healthy"], 3, where)
self.failUnlessEqual(data["count-objects-unhealthy"], 0, where)
@ -582,7 +600,7 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase):
def json_full_deepcheck_and_repair_is_healthy(self, data, n, where):
self.failUnlessEqual(data["root-storage-index"],
base32.b2a(n.get_storage_index()), where)
str(base32.b2a(n.get_storage_index()), "ascii"), where)
self.failUnlessEqual(data["count-objects-checked"], 3, where)
self.failUnlessEqual(data["count-objects-healthy-pre-repair"], 3, where)
@ -728,6 +746,8 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase):
def do_test_cli_good(self, ignored):
d = defer.succeed(None)
if PY3: # TODO fixme once Python 3 CLI porting is done
return d
d.addCallback(lambda ign: self.do_cli_manifest_stream1())
d.addCallback(lambda ign: self.do_cli_manifest_stream2())
d.addCallback(lambda ign: self.do_cli_manifest_stream3())
@ -738,7 +758,7 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase):
return d
def _check_manifest_storage_index(self, out):
lines = [l for l in out.split("\n") if l]
lines = [l for l in out.split(b"\n") if l]
self.failUnlessEqual(len(lines), 3)
self.failUnless(base32.b2a(self.root.get_storage_index()) in lines)
self.failUnless(base32.b2a(self.mutable.get_storage_index()) in lines)
@ -749,7 +769,7 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase):
def _check(args):
(rc, out, err) = args
self.failUnlessEqual(err, "")
lines = [l for l in out.split("\n") if l]
lines = [l for l in out.split(b"\n") if l]
self.failUnlessEqual(len(lines), 8)
caps = {}
for l in lines:
@ -794,7 +814,7 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase):
def _check(args):
(rc, out, err) = args
self.failUnlessEqual(err, "")
lines = [l for l in out.split("\n") if l]
lines = [l for l in out.split(b"\n") if l]
self.failUnlessEqual(len(lines), 3)
self.failUnless(self.root.get_verify_cap().to_string() in lines)
self.failUnless(self.mutable.get_verify_cap().to_string() in lines)
@ -807,7 +827,7 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase):
def _check(args):
(rc, out, err) = args
self.failUnlessEqual(err, "")
lines = [l for l in out.split("\n") if l]
lines = [l for l in out.split(b"\n") if l]
self.failUnlessEqual(len(lines), 3)
self.failUnless(self.root.get_repair_cap().to_string() in lines)
self.failUnless(self.mutable.get_repair_cap().to_string() in lines)
@ -819,7 +839,7 @@ class DeepCheckWebGood(DeepCheckBase, unittest.TestCase):
d = self.do_cli("stats", self.root_uri)
def _check3(args):
(rc, out, err) = args
lines = [l.strip() for l in out.split("\n") if l]
lines = [l.strip() for l in out.split(b"\n") if l]
self.failUnless("count-immutable-files: 1" in lines)
self.failUnless("count-mutable-files: 1" in lines)
self.failUnless("count-literal-files: 3" in lines)
@ -905,17 +925,17 @@ class DeepCheckWebBad(DeepCheckBase, unittest.TestCase):
d.addCallback(self.create_mangled, "large-unrecoverable")
d.addCallback(lambda ignored: c0.create_dirnode())
d.addCallback(self._stash_node, "broken")
large1 = upload.Data("Lots of data\n" * 1000 + "large1" + "\n", None)
large1 = upload.Data(b"Lots of data\n" * 1000 + b"large1" + b"\n", None)
d.addCallback(lambda ignored:
self.nodes["broken"].add_file(u"large1", large1))
d.addCallback(lambda ignored:
self.nodes["broken"].create_subdirectory(u"subdir-good"))
large2 = upload.Data("Lots of data\n" * 1000 + "large2" + "\n", None)
large2 = upload.Data(b"Lots of data\n" * 1000 + b"large2" + b"\n", None)
d.addCallback(lambda subdir: subdir.add_file(u"large2-good", large2))
d.addCallback(lambda ignored:
self.nodes["broken"].create_subdirectory(u"subdir-unrecoverable"))
d.addCallback(self._stash_node, "subdir-unrecoverable")
large3 = upload.Data("Lots of data\n" * 1000 + "large3" + "\n", None)
large3 = upload.Data(b"Lots of data\n" * 1000 + b"large3" + b"\n", None)
d.addCallback(lambda subdir: subdir.add_file(u"large3-good", large3))
d.addCallback(lambda ignored:
self._delete_most_shares(self.nodes["broken"]))
@ -928,14 +948,14 @@ class DeepCheckWebBad(DeepCheckBase, unittest.TestCase):
def create_mangled(self, ignored, name):
nodetype, mangletype = name.split("-", 1)
if nodetype == "mutable":
mutable_uploadable = MutableData("mutable file contents")
mutable_uploadable = MutableData(b"mutable file contents")
d = self.g.clients[0].create_mutable_file(mutable_uploadable)
d.addCallback(lambda n: self.root.set_node(str(name), n))
d.addCallback(lambda n: self.root.set_node(str(name), n)) # TODO drop str() once strings are unicode
elif nodetype == "large":
large = upload.Data("Lots of data\n" * 1000 + name + "\n", None)
large = upload.Data(b"Lots of data\n" * 1000 + name.encode("ascii") + b"\n", None)
d = self.root.add_file(str(name), large)
elif nodetype == "small":
small = upload.Data("Small enough for a LIT", None)
small = upload.Data(b"Small enough for a LIT", None)
d = self.root.add_file(str(name), small)
d.addCallback(self._stash_node, name)
@ -959,10 +979,10 @@ class DeepCheckWebBad(DeepCheckBase, unittest.TestCase):
def _corrupt_some_shares(self, node):
for (shnum, serverid, sharefile) in self.find_uri_shares(node.get_uri()):
if shnum in (0,1):
yield run_cli("debug", "corrupt-share", native_str(sharefile))
yield run_cli("debug", "corrupt-share", sharefile)
def _delete_most_shares(self, node):
self.delete_shares_numbered(node.get_uri(), range(1,10))
self.delete_shares_numbered(node.get_uri(), list(range(1,10)))
def check_is_healthy(self, cr, where):
@ -1199,11 +1219,11 @@ class Large(DeepCheckBase, unittest.TestCase):
self.subdir_node = subdir_node
kids = {}
for i in range(1, COUNT):
litcap = LiteralFileURI("%03d-data" % i).to_string()
litcap = LiteralFileURI(b"%03d-data" % i).to_string()
kids[u"%03d-small" % i] = (litcap, litcap)
return subdir_node.set_children(kids)
d.addCallback(_add_children)
up = upload.Data("large enough for CHK" * 100, "")
up = upload.Data(b"large enough for CHK" * 100, b"")
d.addCallback(lambda ign: self.subdir_node.add_file(u"0000-large", up))
def _start_deepcheck(ignored):

9
src/allmydata/util/_python3.py
View File

@ -29,6 +29,7 @@ PORTED_MODULES = [
"allmydata._monkeypatch",
"allmydata.blacklist",
"allmydata.check_results",
"allmydata.client",
"allmydata.codec",
"allmydata.control",
"allmydata.crypto",
@ -88,6 +89,7 @@ PORTED_MODULES = [
"allmydata.storage.server",
"allmydata.storage.shares",
"allmydata.test.no_network",
"allmydata.test.matchers",
"allmydata.test.mutable.util",
"allmydata.unknown",
"allmydata.uri",
@ -159,12 +161,18 @@ PORTED_TEST_MODULES = [
"allmydata.test.test_base32",
"allmydata.test.test_base62",
"allmydata.test.test_checker",
"allmydata.test.test_client",
"allmydata.test.test_codec",
"allmydata.test.test_common_util",
"allmydata.test.test_configutil",
"allmydata.test.test_connection_status",
"allmydata.test.test_crawler",
"allmydata.test.test_crypto",
# Only partially ported, CLI-using test code is disabled for now until CLI
# is ported.
"allmydata.test.test_deepcheck",
"allmydata.test.test_deferredutil",
"allmydata.test.test_dictutil",
"allmydata.test.test_dirnode",
@ -219,4 +227,5 @@ PORTED_TEST_MODULES = [
"allmydata.test.web.test_util",
"allmydata.test.web.test_web",
"allmydata.test.web.test_webish",
"allmydata.test.test_windows",
]

10
src/allmydata/util/eliotutil.py
View File

@ -32,7 +32,7 @@ from six import ensure_text
from sys import (
stdout,
)
from functools import wraps
from functools import wraps, partial
from logging import (
INFO,
Handler,
@ -66,6 +66,7 @@ from eliot.twisted import (
DeferredContext,
inline_callbacks,
)
from eliot.testing import capture_logging as eliot_capture_logging
from twisted.python.usage import (
UsageError,
@ -326,3 +327,10 @@ def log_call_deferred(action_type):
return DeferredContext(d).addActionFinish()
return logged_f
return decorate_log_call_deferred
# On Python 3, encoding bytes to JSON doesn't work, so we have a custom JSON
# encoder we want to use when validating messages.
if PY2:
capture_logging = eliot_capture_logging
else:
capture_logging = partial(eliot_capture_logging, encoder_=BytesJSONEncoder)