From 9d64c881998b0cdb6962fd8abe18b86d73e31c39 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 22 Mar 2021 20:28:18 -0400 Subject: [PATCH 01/65] news fragment --- newsfragments/3645.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3645.minor diff --git a/newsfragments/3645.minor b/newsfragments/3645.minor new file mode 100644 index 000000000..e69de29bb From 31c838d5848eec9a65e63a63bb01b9a24446cb42 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 22 Mar 2021 20:28:24 -0400 Subject: [PATCH 02/65] the expanded motivation --- docs/proposed/http-storage-node-protocol.rst | 48 ++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index 36c718c56..17da870b1 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -13,6 +13,54 @@ Specifically, it should be possible to implement a Tahoe-LAFS storage server wit The Tahoe-LAFS client will also need to change but it is not expected that it will be noticably simplified by this change (though this may be the first step towards simplifying it). +Motivation +---------- + +Foolscap +~~~~~~~~ + +Foolscap is a remote method invocation protocol with several distinctive features. +At its core it allows separate processes to refer each other's objects and methods using a capability-based model. +This allows for extremely fine-grained access control in a system that remains highly securable without becoming overwhelmingly complicated. +Supporting this is a flexible and extensible serialization system which allows data to be exchanged between processes in carefully controlled ways. + +Tahoe avails itself of only a small portion of these features. +A Tahoe storage server typically only exposes one object with a fixed set of methods to clients. +A Tahoe introducer node does roughly the same. +Tahoe exchanges simple data structures that have many common, standard serialized representations. + +In exchange for this slight use of Foolscap's sophisticated mechanisms, +Tahoe pays a substantial price: + +* Foolscap is implemented only for Python. + Tahoe is thus limited to being implemented on in Python. +* There is only one Python implementation of Foolscap. + The implementation is therefore the de facto standard and understanding of the protocol often relies on understanding that implementation. +* The Foolscap developer community is very small. + The implementation therefore advances very little and some non-trivial part of the maintenance cost falls on the Tahoe project. +* The extensible serialization system imposes substantial overhead for the simple data structures Tahoe exchanges. + Tahoe therefore presents a more sluggish experience to users and taxes servers more greatly than is necessary. + +HTTP +~~~~ + +HTTP is a request/response protocol that has become the lingua franca of the internet. +Combined with the principles of Representational state transfer (REST) it is widely employed to create, update, and delete data in collections on the internet. +HTTP itself provides only modest functionality in comparison to Foolscap. +However its simplicity and widespread use have led to a diverse and almost overwhelming ecosystem of libraries, frameworks, toolkits, and so on. + +By adopting HTTP in place of Foolscap Tahoe can realize the following concrete benefits: + +* Practically every language or runtime has an HTTP protocol implementation (or a dozen of them) available. + This change paves the way for new Tahoe implementations using tools better suited for certain situations + (mobile client implementations, high-performance server implementations, easily distributed desktop clients, etc). +* The simplicity of and vast quantity of resources about HTTP make it a very easy protocol to learn and use. + This change reduces the barrier to entry for developers to contribute improvements to Tahoe's network interactions. +* For any given language there is very likely an HTTP implementation with a large and active developer community. + Tahoe can therefore benefit from the large effort being put into making better libraries for using HTTP. +* One of the core features of HTTP is the mundane transfer of bulk data and implementions are often capable of doing this with extreme efficiency. + The alignment of this core feature with a core activity of Tahoe of transferring bulk data means that a substantial barrier to improved Tahoe runtime performance will be eliminated. + Requirements ------------ From 835c050e6c5cee9cb272bad3de3f17e2e8c9d68a Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 30 Mar 2021 12:39:22 -0400 Subject: [PATCH 03/65] fix word-o --- docs/proposed/http-storage-node-protocol.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index 8d65e121d..5e46480d3 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -33,7 +33,7 @@ In exchange for this slight use of Foolscap's sophisticated mechanisms, Tahoe pays a substantial price: * Foolscap is implemented only for Python. - Tahoe is thus limited to being implemented on in Python. + Tahoe is thus limited to being implemented only in Python. * There is only one Python implementation of Foolscap. The implementation is therefore the de facto standard and understanding of the protocol often relies on understanding that implementation. * The Foolscap developer community is very small. From e843206524b5904f8a49ce3ed6614da67db4ca49 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 30 Mar 2021 12:39:27 -0400 Subject: [PATCH 04/65] expand on justification for performance complaints --- docs/proposed/http-storage-node-protocol.rst | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index 5e46480d3..c2a68b3ee 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -39,7 +39,10 @@ Tahoe pays a substantial price: * The Foolscap developer community is very small. The implementation therefore advances very little and some non-trivial part of the maintenance cost falls on the Tahoe project. * The extensible serialization system imposes substantial overhead for the simple data structures Tahoe exchanges. - Tahoe therefore presents a more sluggish experience to users and taxes servers more greatly than is necessary. +* Foolscap encourages a "remote object" style of protocol design with involves many client-server interactions. + However, Foolscap does not implement "promise pipelining". + The result is that Foolscap encourages a protocol that requires many round-trips between client and server. +* The serialization overhead combined with the many round-trips result in Tahoe presenting a more sluggish experience to users and taxes servers more greatly than is necessary. HTTP ~~~~ From 0f0398aeaf24ac7d2ea54c6fc9f06661f552f45a Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 30 Mar 2021 12:39:50 -0400 Subject: [PATCH 05/65] minor typography --- docs/proposed/http-storage-node-protocol.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index c2a68b3ee..6a2f5e933 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -48,7 +48,7 @@ HTTP ~~~~ HTTP is a request/response protocol that has become the lingua franca of the internet. -Combined with the principles of Representational state transfer (REST) it is widely employed to create, update, and delete data in collections on the internet. +Combined with the principles of Representational State Transfer (REST) it is widely employed to create, update, and delete data in collections on the internet. HTTP itself provides only modest functionality in comparison to Foolscap. However its simplicity and widespread use have led to a diverse and almost overwhelming ecosystem of libraries, frameworks, toolkits, and so on. From 94b92202c3f448561318f03f9b070eea5f14ef2a Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 30 Mar 2021 12:45:22 -0400 Subject: [PATCH 06/65] Try to clarify requirement for some security properties --- docs/proposed/http-storage-node-protocol.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index 6a2f5e933..a760ed7d9 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -64,6 +64,14 @@ By adopting HTTP in place of Foolscap Tahoe can realize the following concrete b * One of the core features of HTTP is the mundane transfer of bulk data and implementions are often capable of doing this with extreme efficiency. The alignment of this core feature with a core activity of Tahoe of transferring bulk data means that a substantial barrier to improved Tahoe runtime performance will be eliminated. +TLS +~~~ + +The Foolscap-based protocol provides *some* of Tahoe's confidentiality, integrity, and authentication properties by leveraging TLS. +An HTTP-based protocol can make use of TLS in largely the same way to provide the same properties. +Provision of these properties *is* dependant on implementers following Great Black Swamp's rules for x509 certificate validation +(rather than the standard "web" rules for validation). + Requirements ------------ From fbcb9bef2922c7b9406daa706253effcbb95b0e5 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 31 Mar 2021 10:09:54 -0400 Subject: [PATCH 07/65] Delete unused code. --- src/allmydata/util/sibpath.py | 24 --- src/allmydata/util/verlib.py | 336 ---------------------------------- 2 files changed, 360 deletions(-) delete mode 100644 src/allmydata/util/sibpath.py delete mode 100644 src/allmydata/util/verlib.py diff --git a/src/allmydata/util/sibpath.py b/src/allmydata/util/sibpath.py deleted file mode 100644 index 80a2801a3..000000000 --- a/src/allmydata/util/sibpath.py +++ /dev/null @@ -1,24 +0,0 @@ -import os -import sys -from twisted.python.util import sibpath as tsibpath - -def sibpath(path, sibling): - """ - Looks for a named sibling relative to the given path. If such a file - exists, its path will be returned, otherwise a second search will be - made for the named sibling relative to the path of the executable - currently running. This is useful in the case that something built - with py2exe, for example, needs to find data files relative to its - install. Note hence that care should be taken not to search for - private package files whose names might collide with files which might - be found installed alongside the python interpreter itself. If no - file is found in either place, the sibling relative to the given path - is returned, likely leading to a file not found error. - """ - sib = tsibpath(path, sibling) - if not os.path.exists(sib): - exe_sib = tsibpath(sys.executable, sibling) - if os.path.exists(exe_sib): - return exe_sib - return sib - diff --git a/src/allmydata/util/verlib.py b/src/allmydata/util/verlib.py deleted file mode 100644 index 2dfc24a1b..000000000 --- a/src/allmydata/util/verlib.py +++ /dev/null @@ -1,336 +0,0 @@ -""" -"Rational" version definition and parsing for DistutilsVersionFight -discussion at PyCon 2009. - -Ported to Python 3. -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals - -from future.utils import PY2 -if PY2: - from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 - -import re - - -class IrrationalVersionError(Exception): - """This is an irrational version.""" - pass - -class HugeMajorVersionNumError(IrrationalVersionError): - """An irrational version because the major version number is huge - (often because a year or date was used). - - See `error_on_huge_major_num` option in `NormalizedVersion` for details. - This guard can be disabled by setting that option False. - """ - pass - -# A marker used in the second and third parts of the `parts` tuple, for -# versions that don't have those segments, to sort properly. An example -# of versions in sort order ('highest' last): -# 1.0b1 ((1,0), ('b',1), ('f',)) -# 1.0.dev345 ((1,0), ('f',), ('dev', 345)) -# 1.0 ((1,0), ('f',), ('f',)) -# 1.0.post256.dev345 ((1,0), ('f',), ('f', 'post', 256, 'dev', 345)) -# 1.0.post345 ((1,0), ('f',), ('f', 'post', 345, 'f')) -# ^ ^ ^ -# 'b' < 'f' ---------------------/ | | -# | | -# 'dev' < 'f' < 'post' -------------------/ | -# | -# 'dev' < 'f' ----------------------------------------------/ -# Other letters would do, but 'f' for 'final' is kind of nice. -FINAL_MARKER = ('f',) - -VERSION_RE = re.compile(r''' - ^ - (?P\d+\.\d+) # minimum 'N.N' - (?P(?:\.\d+)*) # any number of extra '.N' segments - (?: - (?P[abc]|rc) # 'a'=alpha, 'b'=beta, 'c'=release candidate - # 'rc'= alias for release candidate - (?P\d+(?:\.\d+)*) - )? - (?P(\.post(?P\d+))?(\.dev(?P\d+))?)? - $''', re.VERBOSE) - -class NormalizedVersion(object): - """A rational version. - - Good: - 1.2 # equivalent to "1.2.0" - 1.2.0 - 1.2a1 - 1.2.3a2 - 1.2.3b1 - 1.2.3c1 - 1.2.3.4 - TODO: fill this out - - Bad: - 1 # mininum two numbers - 1.2a # release level must have a release serial - 1.2.3b - """ - def __init__(self, s, error_on_huge_major_num=True): - """Create a NormalizedVersion instance from a version string. - - @param s {str} The version string. - @param error_on_huge_major_num {bool} Whether to consider an - apparent use of a year or full date as the major version number - an error. Default True. One of the observed patterns on PyPI before - the introduction of `NormalizedVersion` was version numbers like this: - 2009.01.03 - 20040603 - 2005.01 - This guard is here to strongly encourage the package author to - use an alternate version, because a release deployed into PyPI - and, e.g. downstream Linux package managers, will forever remove - the possibility of using a version number like "1.0" (i.e. - where the major number is less than that huge major number). - """ - self._parse(s, error_on_huge_major_num) - - @classmethod - def from_parts(cls, version, prerelease=FINAL_MARKER, - devpost=FINAL_MARKER): - return cls(cls.parts_to_str((version, prerelease, devpost))) - - def _parse(self, s, error_on_huge_major_num=True): - """Parses a string version into parts.""" - match = VERSION_RE.search(s) - if not match: - raise IrrationalVersionError(s) - - groups = match.groupdict() - parts = [] - - # main version - block = self._parse_numdots(groups['version'], s, False, 2) - extraversion = groups.get('extraversion') - if extraversion not in ('', None): - block += self._parse_numdots(extraversion[1:], s) - parts.append(tuple(block)) - - # prerelease - prerel = groups.get('prerel') - if prerel is not None: - block = [prerel] - block += self._parse_numdots(groups.get('prerelversion'), s, - pad_zeros_length=1) - parts.append(tuple(block)) - else: - parts.append(FINAL_MARKER) - - # postdev - if groups.get('postdev'): - post = groups.get('post') - dev = groups.get('dev') - postdev = [] - if post is not None: - postdev.extend([FINAL_MARKER[0], 'post', int(post)]) - if dev is None: - postdev.append(FINAL_MARKER[0]) - if dev is not None: - postdev.extend(['dev', int(dev)]) - parts.append(tuple(postdev)) - else: - parts.append(FINAL_MARKER) - self.parts = tuple(parts) - if error_on_huge_major_num and self.parts[0][0] > 1980: - raise HugeMajorVersionNumError("huge major version number, %r, " - "which might cause future problems: %r" % (self.parts[0][0], s)) - - def _parse_numdots(self, s, full_ver_str, drop_trailing_zeros=True, - pad_zeros_length=0): - """Parse 'N.N.N' sequences, return a list of ints. - - @param s {str} 'N.N.N...' sequence to be parsed - @param full_ver_str {str} The full version string from which this - comes. Used for error strings. - @param drop_trailing_zeros {bool} Whether to drop trailing zeros - from the returned list. Default True. - @param pad_zeros_length {int} The length to which to pad the - returned list with zeros, if necessary. Default 0. - """ - nums = [] - for n in s.split("."): - if len(n) > 1 and n[0] == '0': - raise IrrationalVersionError("cannot have leading zero in " - "version number segment: '%s' in %r" % (n, full_ver_str)) - nums.append(int(n)) - if drop_trailing_zeros: - while nums and nums[-1] == 0: - nums.pop() - while len(nums) < pad_zeros_length: - nums.append(0) - return nums - - def __str__(self): - return self.parts_to_str(self.parts) - - @classmethod - def parts_to_str(cls, parts): - """Transforms a version expressed in tuple into its string - representation.""" - # XXX This doesn't check for invalid tuples - main, prerel, postdev = parts - s = '.'.join(str(v) for v in main) - if prerel is not FINAL_MARKER: - s += prerel[0] - s += '.'.join(str(v) for v in prerel[1:]) - if postdev and postdev is not FINAL_MARKER: - if postdev[0] == 'f': - postdev = postdev[1:] - i = 0 - while i < len(postdev): - if i % 2 == 0: - s += '.' - s += str(postdev[i]) - i += 1 - return s - - def __repr__(self): - return "%s('%s')" % (self.__class__.__name__, self) - - def _cannot_compare(self, other): - raise TypeError("cannot compare %s and %s" - % (type(self).__name__, type(other).__name__)) - - def __eq__(self, other): - if not isinstance(other, NormalizedVersion): - self._cannot_compare(other) - return self.parts == other.parts - - def __lt__(self, other): - if not isinstance(other, NormalizedVersion): - self._cannot_compare(other) - return self.parts < other.parts - - def __ne__(self, other): - return not self.__eq__(other) - - def __gt__(self, other): - return not (self.__lt__(other) or self.__eq__(other)) - - def __le__(self, other): - return self.__eq__(other) or self.__lt__(other) - - def __ge__(self, other): - return self.__eq__(other) or self.__gt__(other) - -def suggest_normalized_version(s): - """Suggest a normalized version close to the given version string. - - If you have a version string that isn't rational (i.e. NormalizedVersion - doesn't like it) then you might be able to get an equivalent (or close) - rational version from this function. - - This does a number of simple normalizations to the given string, based - on observation of versions currently in use on PyPI. Given a dump of - those version during PyCon 2009, 4287 of them: - - 2312 (53.93%) match NormalizedVersion without change - - with the automatic suggestion - - 3474 (81.04%) match when using this suggestion method - - @param s {str} An irrational version string. - @returns A rational version string, or None, if couldn't determine one. - """ - try: - NormalizedVersion(s) - return s # already rational - except IrrationalVersionError: - pass - - rs = s.lower() - - # part of this could use maketrans - for orig, repl in (('-alpha', 'a'), ('-beta', 'b'), ('alpha', 'a'), - ('beta', 'b'), ('rc', 'c'), ('-final', ''), - ('-pre', 'c'), - ('-release', ''), ('.release', ''), ('-stable', ''), - ('+', '.'), ('_', '.'), (' ', ''), ('.final', ''), - ('final', '')): - rs = rs.replace(orig, repl) - - # if something ends with dev or pre, we add a 0 - rs = re.sub(r"pre$", r"pre0", rs) - rs = re.sub(r"dev$", r"dev0", rs) - - # if we have something like "b-2" or "a.2" at the end of the - # version, that is pobably beta, alpha, etc - # let's remove the dash or dot - rs = re.sub(r"([abc]|rc)[\-\.](\d+)$", r"\1\2", rs) - - # 1.0-dev-r371 -> 1.0.dev371 - # 0.1-dev-r79 -> 0.1.dev79 - rs = re.sub(r"[\-\.](dev)[\-\.]?r?(\d+)$", r".\1\2", rs) - - # Clean: 2.0.a.3, 2.0.b1, 0.9.0~c1 - rs = re.sub(r"[.~]?([abc])\.?", r"\1", rs) - - # Clean: v0.3, v1.0 - if rs.startswith('v'): - rs = rs[1:] - - # Clean leading '0's on numbers. - #TODO: unintended side-effect on, e.g., "2003.05.09" - # PyPI stats: 77 (~2%) better - rs = re.sub(r"\b0+(\d+)(?!\d)", r"\1", rs) - - # Clean a/b/c with no version. E.g. "1.0a" -> "1.0a0". Setuptools infers - # zero. - # PyPI stats: 245 (7.56%) better - rs = re.sub(r"(\d+[abc])$", r"\g<1>0", rs) - - # the 'dev-rNNN' tag is a dev tag - rs = re.sub(r"\.?(dev-r|dev\.r)\.?(\d+)$", r".dev\2", rs) - - # clean the - when used as a pre delimiter - rs = re.sub(r"-(a|b|c)(\d+)$", r"\1\2", rs) - - # a terminal "dev" or "devel" can be changed into ".dev0" - rs = re.sub(r"[\.\-](dev|devel)$", r".dev0", rs) - - # a terminal "dev" can be changed into ".dev0" - rs = re.sub(r"(?![\.\-])dev$", r".dev0", rs) - - # a terminal "final" or "stable" can be removed - rs = re.sub(r"(final|stable)$", "", rs) - - # The 'r' and the '-' tags are post release tags - # 0.4a1.r10 -> 0.4a1.post10 - # 0.9.33-17222 -> 0.9.33.post17222 - # 0.9.33-r17222 -> 0.9.33.post17222 - rs = re.sub(r"\.?(r|-|-r)\.?(\d+)$", r".post\2", rs) - - # Clean 'r' instead of 'dev' usage: - # 0.9.33+r17222 -> 0.9.33.dev17222 - # 1.0dev123 -> 1.0.dev123 - # 1.0.git123 -> 1.0.dev123 - # 1.0.bzr123 -> 1.0.dev123 - # 0.1a0dev.123 -> 0.1a0.dev123 - # PyPI stats: ~150 (~4%) better - rs = re.sub(r"\.?(dev|git|bzr)\.?(\d+)$", r".dev\2", rs) - - # Clean '.pre' (normalized from '-pre' above) instead of 'c' usage: - # 0.2.pre1 -> 0.2c1 - # 0.2-c1 -> 0.2c1 - # 1.0preview123 -> 1.0c123 - # PyPI stats: ~21 (0.62%) better - rs = re.sub(r"\.?(pre|preview|-c)(\d+)$", r"c\g<2>", rs) - - - # Tcl/Tk uses "px" for their post release markers - rs = re.sub(r"p(\d+)$", r".post\1", rs) - - try: - NormalizedVersion(rs) - return rs # already rational - except IrrationalVersionError: - pass - return None From 1ead68d06128960c16dfda77413cafc4045e79d6 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 31 Mar 2021 10:10:05 -0400 Subject: [PATCH 08/65] Start of tests for consumer.py. --- src/allmydata/test/test_consumer.py | 80 +++++++++++++++++++++++++++++ src/allmydata/util/_python3.py | 1 + 2 files changed, 81 insertions(+) create mode 100644 src/allmydata/test/test_consumer.py diff --git a/src/allmydata/test/test_consumer.py b/src/allmydata/test/test_consumer.py new file mode 100644 index 000000000..98f419764 --- /dev/null +++ b/src/allmydata/test/test_consumer.py @@ -0,0 +1,80 @@ +""" +Tests for allmydata.util.consumer. + +Ported to Python 3. +""" + +from __future__ import unicode_literals +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + +from zope.interface import implementer +from twisted.trial.unittest import TestCase +from twisted.internet.interfaces import IPushProducer, IPullProducer + +from allmydata.util.consumer import MemoryConsumer, download_to_data + + +@implementer(IPushProducer) +@implementer(IPullProducer) +class Producer: + """Can be used as either streaming or non-streaming producer. + + If used as streaming, the test should call iterate() manually. + """ + + def __init__(self, consumer): + self.data = [b"abc", b"def", b"ghi"] + self.consumer = consumer + self.done = False + + def resumeProducing(self): + """Kick off streaming.""" + self.iterate() + + def iterate(self): + """Do another iteration of writing.""" + if self.done: + raise RuntimeError( + "There's a bug somewhere, shouldn't iterate after being done" + ) + if self.data: + self.consumer.write(self.data.pop(0)) + else: + self.done = True + self.consumer.unregisterProducer() + + +class MemoryConsumerTests(TestCase): + """Tests for MemoryConsumer.""" + + def test_push_producer(self): + """ + A MemoryConsumer accumulates all data sent by a streaming producer. + """ + consumer = MemoryConsumer() + producer = Producer(consumer) + consumer.registerProducer(producer, True) + self.assertEqual(consumer.chunks, [b"abc"]) + producer.iterate() + producer.iterate() + self.assertEqual(consumer.chunks, [b"abc", b"def", b"ghi"]) + self.assertEqual(consumer.done, False) + producer.iterate() + self.assertEqual(consumer.chunks, [b"abc", b"def", b"ghi"]) + self.assertEqual(consumer.done, True) + + def test_pull_producer(self): + """ + A MemoryConsumer accumulates all data sent by a non-streaming producer. + """ + consumer = MemoryConsumer() + producer = Producer(consumer) + consumer.registerProducer(producer, False) + self.assertEqual(consumer.chunks, [b"abc", b"def", b"ghi"]) + self.assertEqual(consumer.done, True) diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 2f6857cad..61c444dc1 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -191,6 +191,7 @@ PORTED_TEST_MODULES = [ "allmydata.test.test_configutil", "allmydata.test.test_connections", "allmydata.test.test_connection_status", + "allmydata.test.test_consumer", "allmydata.test.test_crawler", "allmydata.test.test_crypto", From 1ef33d3d69de4a66ef70eb9451ddeb149ef69c7f Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 31 Mar 2021 10:50:50 -0400 Subject: [PATCH 09/65] Note download_to_data() is tested elsewhere. --- src/allmydata/test/test_consumer.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/allmydata/test/test_consumer.py b/src/allmydata/test/test_consumer.py index 98f419764..b6c1d1d28 100644 --- a/src/allmydata/test/test_consumer.py +++ b/src/allmydata/test/test_consumer.py @@ -78,3 +78,7 @@ class MemoryConsumerTests(TestCase): consumer.registerProducer(producer, False) self.assertEqual(consumer.chunks, [b"abc", b"def", b"ghi"]) self.assertEqual(consumer.done, True) + + +# download_to_data() is effectively tested by some of the filenode tests, e.g. +# test_immutable.py. From 8439f2820bf00902b185f749c36f5ea7ec9cd643 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 31 Mar 2021 10:53:02 -0400 Subject: [PATCH 10/65] Port to Python 3. --- src/allmydata/util/_python3.py | 1 + src/allmydata/util/consumer.py | 18 +++++++++++++++--- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 61c444dc1..e3fee8dc6 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -115,6 +115,7 @@ PORTED_MODULES = [ "allmydata.util.base62", "allmydata.util.configutil", "allmydata.util.connection_status", + "allmydata.util.consumer", "allmydata.util.deferredutil", "allmydata.util.dictutil", "allmydata.util.eliotutil", diff --git a/src/allmydata/util/consumer.py b/src/allmydata/util/consumer.py index a8eededcc..3de82974d 100644 --- a/src/allmydata/util/consumer.py +++ b/src/allmydata/util/consumer.py @@ -1,11 +1,22 @@ - -"""This file defines a basic download-to-memory consumer, suitable for use in -a filenode's read() method. See download_to_data() for an example of its use. """ +This file defines a basic download-to-memory consumer, suitable for use in +a filenode's read() method. See download_to_data() for an example of its use. + +Ported to Python 3. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from zope.interface import implementer from twisted.internet.interfaces import IConsumer + @implementer(IConsumer) class MemoryConsumer(object): @@ -28,6 +39,7 @@ class MemoryConsumer(object): def unregisterProducer(self): self.done = True + def download_to_data(n, offset=0, size=None): """ Return Deferred that fires with results of reading from the given filenode. From 94358d4587e28999cfa306fe32db2a9fbbf61acd Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Wed, 31 Mar 2021 15:54:42 -0400 Subject: [PATCH 11/65] Add newsfragment --- newsfragments/3616.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3616.minor diff --git a/newsfragments/3616.minor b/newsfragments/3616.minor new file mode 100644 index 000000000..e69de29bb From aebbc52f067faf3176809af4736ed162f53db505 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Wed, 31 Mar 2021 15:54:14 -0400 Subject: [PATCH 12/65] Add Python 3.6 to GitHub Actions test matrix Let us send more coverage reports to coveralls.io --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 39cd114d3..c3e789ede 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,6 +22,7 @@ jobs: - windows-latest python-version: - 2.7 + - 3.6 steps: From dd3b95a0bd039ec5b337780f1ecc1adf9040b8cb Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Wed, 31 Mar 2021 15:54:31 -0400 Subject: [PATCH 13/65] Add ubuntu-latest to GitHub Actions matrix --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c3e789ede..c45ceaa63 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,6 +20,7 @@ jobs: os: - macos-latest - windows-latest + - ubuntu-latest python-version: - 2.7 - 3.6 From 1351a62ac4b48a4b9c2a05635c1a676151e63cfb Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Wed, 31 Mar 2021 16:13:41 -0400 Subject: [PATCH 14/65] Expect coverage tests to fail on Python 3.6 + Windows --- .github/workflows/ci.yml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c45ceaa63..3e460e722 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,7 +23,17 @@ jobs: - ubuntu-latest python-version: - 2.7 - - 3.6 + + # Also run coverage tests with Python 3.6. Expect failures on + # Windows, until we've had a chance to deal with them. + include: + - python-version: 3.6 + os: + - macos-latest + - ubuntu-latest + - python-version: 3.6 + os: windows-latest + experimental: true steps: From 2e67e070811f42825f24f238cd813bfe18a2dca0 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Wed, 31 Mar 2021 16:42:39 -0400 Subject: [PATCH 15/65] Just don't run coverage tests with Python 3.6 + Windows Another test matrix setup I tried is this: jobs: coverage: matrix: os: - macos-latest - windows-latest - ubuntu-latest python-version: - 2.7 include: - python-version: 3.6 os: - macos-latest - ubuntu-latest - python-version: 3.6 os: windows-latest experimental: true But that failed on Python 3.6 + macOS with a simple error message, and no further explanation: "This check failed". Huh? Might simply exclude Windows altogether, because that approach sort of worked in another experiment. --- .github/workflows/ci.yml | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3e460e722..dcbab9461 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,17 +23,13 @@ jobs: - ubuntu-latest python-version: - 2.7 - - # Also run coverage tests with Python 3.6. Expect failures on - # Windows, until we've had a chance to deal with them. - include: - - python-version: 3.6 - os: - - macos-latest - - ubuntu-latest + - 3.6 + exclude: + # Do not run coverage tests with Python 3.6 on Windows for + # now. They will fail. Dealing with them separately would + # be simpler. - python-version: 3.6 os: windows-latest - experimental: true steps: From d17f3d36c2e7712f67cceb8b73ebce5899bbb3d6 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Wed, 31 Mar 2021 17:57:44 -0400 Subject: [PATCH 16/65] Run coveralls verbosely --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dcbab9461..4c12c7f7f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -117,7 +117,7 @@ jobs: - name: "Report Coverage to Coveralls" run: | pip3 install --upgrade coveralls==3.0.1 - python3 -m coveralls + python3 -m coveralls --verbose env: # Some magic value required for some magic reason. GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" @@ -152,7 +152,7 @@ jobs: - name: "Indicate completion to coveralls.io" run: | pip3 install --upgrade coveralls==3.0.1 - python3 -m coveralls --finish + python3 -m coveralls --verbose --finish env: # Some magic value required for some magic reason. GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" From 2a620863230e170cd4fdd51e0c509953690ad0b7 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Wed, 31 Mar 2021 19:16:58 -0400 Subject: [PATCH 17/65] Turn down coveralls verbosity --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4c12c7f7f..dcbab9461 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -117,7 +117,7 @@ jobs: - name: "Report Coverage to Coveralls" run: | pip3 install --upgrade coveralls==3.0.1 - python3 -m coveralls --verbose + python3 -m coveralls env: # Some magic value required for some magic reason. GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" @@ -152,7 +152,7 @@ jobs: - name: "Indicate completion to coveralls.io" run: | pip3 install --upgrade coveralls==3.0.1 - python3 -m coveralls --verbose --finish + python3 -m coveralls --finish env: # Some magic value required for some magic reason. GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" From de2609151e2add0b505f62c8644a5e86a79e7117 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 1 Apr 2021 10:01:14 -0400 Subject: [PATCH 18/65] Special logic for roundtripping Unicode to Unicode is only necessary on Python 2. --- src/allmydata/test/test_util.py | 7 ++++++- src/allmydata/util/yamlutil.py | 29 +++++++++++++++++++++++------ 2 files changed, 29 insertions(+), 7 deletions(-) diff --git a/src/allmydata/test/test_util.py b/src/allmydata/test/test_util.py index a14adb787..4dc2793a4 100644 --- a/src/allmydata/test/test_util.py +++ b/src/allmydata/test/test_util.py @@ -480,7 +480,12 @@ class EqButNotIs(object): class YAML(unittest.TestCase): def test_convert(self): - data = yaml.safe_dump(["str", u"unicode", u"\u1234nicode"]) + """ + Unicode and (ASCII) native strings get roundtripped to Unicode strings. + """ + data = yaml.safe_dump( + [six.ensure_str("str"), u"unicode", u"\u1234nicode"] + ) back = yamlutil.safe_load(data) self.assertIsInstance(back[0], str) self.assertIsInstance(back[1], str) diff --git a/src/allmydata/util/yamlutil.py b/src/allmydata/util/yamlutil.py index 40c38fa30..f7eb8004f 100644 --- a/src/allmydata/util/yamlutil.py +++ b/src/allmydata/util/yamlutil.py @@ -1,11 +1,28 @@ +from future.utils import PY2 + import yaml -# Announcements contain unicode, because they come from JSON. We tell PyYAML -# to give us unicode instead of str/bytes. -def construct_unicode(loader, node): - return node.value -yaml.SafeLoader.add_constructor("tag:yaml.org,2002:str", - construct_unicode) +if PY2: + # On Python 2 the way pyyaml deals with Unicode strings is inconsistent. + # + # >>> yaml.safe_load(yaml.safe_dump(u"hello")) + # 'hello' + # >>> yaml.safe_load(yaml.safe_dump(u"hello\u1234")) + # u'hello\u1234' + # + # In other words, Unicode strings get roundtripped to byte strings, but + # only sometimes. + # + # In order to ensure unicode stays unicode, we add a configuration saying + # that the YAML String Language-Independent Type ("a sequence of zero or + # more Unicode characters") should be the underlying Unicode string object, + # rather than converting to bytes when possible. + # + # Reference: https://yaml.org/type/str.html + def construct_unicode(loader, node): + return node.value + yaml.SafeLoader.add_constructor("tag:yaml.org,2002:str", + construct_unicode) def safe_load(f): return yaml.safe_load(f) From 5ac631047c26fee933d6407f209ea0733fad3a99 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 1 Apr 2021 10:04:01 -0400 Subject: [PATCH 19/65] Port to Python 3. --- src/allmydata/util/_python3.py | 1 + src/allmydata/util/yamlutil.py | 11 +++++++++++ 2 files changed, 12 insertions(+) diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index e3fee8dc6..e8783aee7 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -140,6 +140,7 @@ PORTED_MODULES = [ "allmydata.util.statistics", "allmydata.util.time_format", "allmydata.util.tor_provider", + "allmydata.util.yamlutil", "allmydata.web", "allmydata.web.check_results", "allmydata.web.common", diff --git a/src/allmydata/util/yamlutil.py b/src/allmydata/util/yamlutil.py index f7eb8004f..fd9fc73e2 100644 --- a/src/allmydata/util/yamlutil.py +++ b/src/allmydata/util/yamlutil.py @@ -1,7 +1,18 @@ +""" +Ported to Python 3. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import yaml + if PY2: # On Python 2 the way pyyaml deals with Unicode strings is inconsistent. # From f606420d7823b29e5fb359d5bf91c87b0d4006f1 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 1 Apr 2021 10:14:05 -0400 Subject: [PATCH 20/65] Make it new-style. --- src/allmydata/test/test_consumer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/test_consumer.py b/src/allmydata/test/test_consumer.py index b6c1d1d28..46b87e477 100644 --- a/src/allmydata/test/test_consumer.py +++ b/src/allmydata/test/test_consumer.py @@ -22,7 +22,7 @@ from allmydata.util.consumer import MemoryConsumer, download_to_data @implementer(IPushProducer) @implementer(IPullProducer) -class Producer: +class Producer(object): """Can be used as either streaming or non-streaming producer. If used as streaming, the test should call iterate() manually. From 9f02de688ca4baffb1dee7fd3d599d20776c96b7 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 1 Apr 2021 10:19:16 -0400 Subject: [PATCH 21/65] We don't support old Foolscap versions anymore. --- src/allmydata/introducer/common.py | 6 ++++-- src/allmydata/util/rrefutil.py | 13 ------------- 2 files changed, 4 insertions(+), 15 deletions(-) diff --git a/src/allmydata/introducer/common.py b/src/allmydata/introducer/common.py index f67aad203..dd9bb527c 100644 --- a/src/allmydata/introducer/common.py +++ b/src/allmydata/introducer/common.py @@ -11,9 +11,11 @@ if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import re + +from foolscap.api import SturdyRef from allmydata.crypto.util import remove_prefix from allmydata.crypto import ed25519 -from allmydata.util import base32, rrefutil, jsonbytes as json +from allmydata.util import base32, jsonbytes as json def get_tubid_string_from_ann(ann): @@ -127,6 +129,6 @@ class AnnouncementDescriptor(object): self.serverid = key_s furl = ann_d.get("anonymous-storage-FURL") if furl: - self.connection_hints = rrefutil.connection_hints_for_furl(furl) + self.connection_hints = list(SturdyRef(furl).locationHints) else: self.connection_hints = [] diff --git a/src/allmydata/util/rrefutil.py b/src/allmydata/util/rrefutil.py index 40e921507..abcd0e4d2 100644 --- a/src/allmydata/util/rrefutil.py +++ b/src/allmydata/util/rrefutil.py @@ -20,19 +20,6 @@ def add_version_to_remote_reference(rref, default): return d -def connection_hints_for_furl(furl): - hints = [] - for h in SturdyRef(furl).locationHints: - # Foolscap-0.2.5 and earlier used strings in .locationHints, 0.2.6 - # through 0.6.4 used tuples of ("ipv4",host,port), 0.6.5 through - # 0.8.0 used tuples of ("tcp",host,port), and >=0.9.0 uses strings - # again. Tolerate them all. - if isinstance(h, tuple): - hints.append(":".join([str(s) for s in h])) - else: - hints.append(h) - return hints - def stringify_remote_address(rref): remote = rref.getPeer() if isinstance(remote, address.IPv4Address): From 062740dc23ac43a2d34250e5b574609f06936753 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 1 Apr 2021 10:23:37 -0400 Subject: [PATCH 22/65] Probably not worth unit testing a utility function used in one place, move it to place it's used. --- src/allmydata/introducer/server.py | 10 ++++++++++ src/allmydata/util/rrefutil.py | 9 --------- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/src/allmydata/introducer/server.py b/src/allmydata/introducer/server.py index dcc2fd2c0..6d73c4026 100644 --- a/src/allmydata/introducer/server.py +++ b/src/allmydata/introducer/server.py @@ -24,6 +24,7 @@ except ImportError: from zope.interface import implementer from twisted.application import service from twisted.internet import defer +from twisted.internet.address import IPv4Address from twisted.python.failure import Failure from foolscap.api import Referenceable import allmydata @@ -148,6 +149,15 @@ class _IntroducerNode(node.Node): ws = IntroducerWebishServer(self, webport, nodeurl_path, staticdir) ws.setServiceParent(self) + +def stringify_remote_address(rref): + remote = rref.getPeer() + if isinstance(remote, IPv4Address): + return "%s:%d" % (remote.host, remote.port) + # loopback is a non-IPv4Address + return str(remote) + + @implementer(RIIntroducerPublisherAndSubscriberService_v2) class IntroducerService(service.MultiService, Referenceable): name = "introducer" diff --git a/src/allmydata/util/rrefutil.py b/src/allmydata/util/rrefutil.py index abcd0e4d2..de0f2fbd5 100644 --- a/src/allmydata/util/rrefutil.py +++ b/src/allmydata/util/rrefutil.py @@ -1,5 +1,4 @@ -from twisted.internet import address from foolscap.api import Violation, RemoteException, SturdyRef @@ -18,11 +17,3 @@ def add_version_to_remote_reference(rref, default): return rref d.addCallbacks(_got_version, _no_get_version) return d - - -def stringify_remote_address(rref): - remote = rref.getPeer() - if isinstance(remote, address.IPv4Address): - return "%s:%d" % (remote.host, remote.port) - # loopback is a non-IPv4Address - return str(remote) From 6127fc8cc78c9c0c7da492bca48f9a37526dec44 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 1 Apr 2021 10:49:59 -0400 Subject: [PATCH 23/65] Tests for rrefutil. --- src/allmydata/test/test_util.py | 39 +++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/src/allmydata/test/test_util.py b/src/allmydata/test/test_util.py index 4dc2793a4..7b0b52783 100644 --- a/src/allmydata/test/test_util.py +++ b/src/allmydata/test/test_util.py @@ -17,12 +17,15 @@ import yaml import json from twisted.trial import unittest +from twisted.internet import defer +from foolscap.api import Violation, RemoteException from allmydata.util import idlib, mathutil from allmydata.util import fileutil from allmydata.util import jsonbytes from allmydata.util import pollmixin from allmydata.util import yamlutil +from allmydata.util import rrefutil from allmydata.util.fileutil import EncryptedTemporaryFile from allmydata.test.common_util import ReallyEqualMixin @@ -526,3 +529,39 @@ class JSONBytes(unittest.TestCase): encoded = jsonbytes.dumps_bytes(x) self.assertIsInstance(encoded, bytes) self.assertEqual(json.loads(encoded, encoding="utf-8"), x) + + +class FakeRemoteRef(object): + """Emulate a RemoteRef.""" + + def __init__(self, result): + self.result = result + + def callRemote(self, method): + assert method == "get_version" + if isinstance(self.result, Exception): + return defer.fail(self.result) + return defer.succeed(self.result) + + +class RrefUtilTests(unittest.TestCase): + """Tests for rrefutil.""" + + def test_version_returned(self): + """If get_version() succeeded, it is set on the rref.""" + rref = FakeRemoteRef(12345) + result = self.successResultOf( + rrefutil.add_version_to_remote_reference(rref, "default") + ) + self.assertEqual(result.version, 12345) + self.assertIdentical(result, rref) + + def test_exceptions(self): + """If get_version() failed, default version is set on the rref.""" + for exception in (Violation(), RemoteException(ValueError())): + rref = FakeRemoteRef(exception) + result = self.successResultOf( + rrefutil.add_version_to_remote_reference(rref, "Default") + ) + self.assertEqual(result.version, "Default") + self.assertIdentical(result, rref) From e92b88195c6554257602442b83b5a721270811ba Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 1 Apr 2021 10:51:13 -0400 Subject: [PATCH 24/65] Port to Python 3. --- src/allmydata/util/_python3.py | 1 + src/allmydata/util/rrefutil.py | 11 +++++++++++ 2 files changed, 12 insertions(+) diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index e8783aee7..56d9e6aec 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -136,6 +136,7 @@ PORTED_MODULES = [ "allmydata.util.observer", "allmydata.util.pipeline", "allmydata.util.pollmixin", + "allmydata.util.rrefutil", "allmydata.util.spans", "allmydata.util.statistics", "allmydata.util.time_format", diff --git a/src/allmydata/util/rrefutil.py b/src/allmydata/util/rrefutil.py index de0f2fbd5..5d57a2f45 100644 --- a/src/allmydata/util/rrefutil.py +++ b/src/allmydata/util/rrefutil.py @@ -1,3 +1,14 @@ +""" +Ported to Python 3. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from foolscap.api import Violation, RemoteException, SturdyRef From 003e9c62a8106add64564b0b2ec47df1204cd182 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 1 Apr 2021 10:58:27 -0400 Subject: [PATCH 25/65] Delete unused code paths. --- src/allmydata/util/dbutil.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/src/allmydata/util/dbutil.py b/src/allmydata/util/dbutil.py index 543dd2797..11491a4d0 100644 --- a/src/allmydata/util/dbutil.py +++ b/src/allmydata/util/dbutil.py @@ -2,8 +2,6 @@ import os, sys import sqlite3 -from sqlite3 import IntegrityError -[IntegrityError] class DBError(Exception): @@ -12,7 +10,7 @@ class DBError(Exception): def get_db(dbfile, stderr=sys.stderr, create_version=(None, None), updaters={}, just_create=False, dbname="db", - journal_mode=None, synchronous=None): + ): """Open or create the given db file. The parent directory must exist. create_version=(SCHEMA, VERNUM), and SCHEMA must have a 'version' table. Updaters is a {newver: commands} mapping, where e.g. updaters[2] is used @@ -32,12 +30,6 @@ def get_db(dbfile, stderr=sys.stderr, # The default is unspecified according to . c.execute("PRAGMA foreign_keys = ON;") - if journal_mode is not None: - c.execute("PRAGMA journal_mode = %s;" % (journal_mode,)) - - if synchronous is not None: - c.execute("PRAGMA synchronous = %s;" % (synchronous,)) - if must_create: c.executescript(schema) c.execute("INSERT INTO version (version) VALUES (?)", (target_version,)) From fd1860705ee710bcf03bd0196fd0bced9e06ef0f Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 1 Apr 2021 10:58:50 -0400 Subject: [PATCH 26/65] Port to Python 3. --- src/allmydata/util/_python3.py | 1 + src/allmydata/util/dbutil.py | 16 ++++++++++++++++ 2 files changed, 17 insertions(+) diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 56d9e6aec..d5ea3ee31 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -116,6 +116,7 @@ PORTED_MODULES = [ "allmydata.util.configutil", "allmydata.util.connection_status", "allmydata.util.consumer", + "allmydata.util.dbutil", "allmydata.util.deferredutil", "allmydata.util.dictutil", "allmydata.util.eliotutil", diff --git a/src/allmydata/util/dbutil.py b/src/allmydata/util/dbutil.py index 11491a4d0..916382972 100644 --- a/src/allmydata/util/dbutil.py +++ b/src/allmydata/util/dbutil.py @@ -1,3 +1,19 @@ +""" +SQLite3 utilities. + +Test coverage currently provided by test_backupdb.py. + +Ported to Python 3. +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import os, sys From c15fe70378da89e3b50fedeea7db3a5fcafb54c8 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 1 Apr 2021 12:53:25 -0400 Subject: [PATCH 27/65] Replace plain "Tahoe" with "Tahoe-LAFS" as the name of the project/software/system/etc --- docs/proposed/http-storage-node-protocol.rst | 30 ++++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index a760ed7d9..83b6823e7 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -24,25 +24,25 @@ At its core it allows separate processes to refer each other's objects and metho This allows for extremely fine-grained access control in a system that remains highly securable without becoming overwhelmingly complicated. Supporting this is a flexible and extensible serialization system which allows data to be exchanged between processes in carefully controlled ways. -Tahoe avails itself of only a small portion of these features. -A Tahoe storage server typically only exposes one object with a fixed set of methods to clients. -A Tahoe introducer node does roughly the same. -Tahoe exchanges simple data structures that have many common, standard serialized representations. +Tahoe-LAFS avails itself of only a small portion of these features. +A Tahoe-LAFS storage server typically only exposes one object with a fixed set of methods to clients. +A Tahoe-LAFS introducer node does roughly the same. +Tahoe-LAFS exchanges simple data structures that have many common, standard serialized representations. In exchange for this slight use of Foolscap's sophisticated mechanisms, -Tahoe pays a substantial price: +Tahoe-LAFS pays a substantial price: * Foolscap is implemented only for Python. - Tahoe is thus limited to being implemented only in Python. + Tahoe-LAFS is thus limited to being implemented only in Python. * There is only one Python implementation of Foolscap. The implementation is therefore the de facto standard and understanding of the protocol often relies on understanding that implementation. * The Foolscap developer community is very small. - The implementation therefore advances very little and some non-trivial part of the maintenance cost falls on the Tahoe project. -* The extensible serialization system imposes substantial overhead for the simple data structures Tahoe exchanges. + The implementation therefore advances very little and some non-trivial part of the maintenance cost falls on the Tahoe-LAFS project. +* The extensible serialization system imposes substantial overhead for the simple data structures Tahoe-LAFS exchanges. * Foolscap encourages a "remote object" style of protocol design with involves many client-server interactions. However, Foolscap does not implement "promise pipelining". The result is that Foolscap encourages a protocol that requires many round-trips between client and server. -* The serialization overhead combined with the many round-trips result in Tahoe presenting a more sluggish experience to users and taxes servers more greatly than is necessary. +* The serialization overhead combined with the many round-trips result in Tahoe-LAFS presenting a more sluggish experience to users and taxes servers more greatly than is necessary. HTTP ~~~~ @@ -52,22 +52,22 @@ Combined with the principles of Representational State Transfer (REST) it is wid HTTP itself provides only modest functionality in comparison to Foolscap. However its simplicity and widespread use have led to a diverse and almost overwhelming ecosystem of libraries, frameworks, toolkits, and so on. -By adopting HTTP in place of Foolscap Tahoe can realize the following concrete benefits: +By adopting HTTP in place of Foolscap Tahoe-LAFS can realize the following concrete benefits: * Practically every language or runtime has an HTTP protocol implementation (or a dozen of them) available. - This change paves the way for new Tahoe implementations using tools better suited for certain situations + This change paves the way for new Tahoe-LAFS implementations using tools better suited for certain situations (mobile client implementations, high-performance server implementations, easily distributed desktop clients, etc). * The simplicity of and vast quantity of resources about HTTP make it a very easy protocol to learn and use. - This change reduces the barrier to entry for developers to contribute improvements to Tahoe's network interactions. + This change reduces the barrier to entry for developers to contribute improvements to Tahoe-LAFS's network interactions. * For any given language there is very likely an HTTP implementation with a large and active developer community. - Tahoe can therefore benefit from the large effort being put into making better libraries for using HTTP. + Tahoe-LAFS can therefore benefit from the large effort being put into making better libraries for using HTTP. * One of the core features of HTTP is the mundane transfer of bulk data and implementions are often capable of doing this with extreme efficiency. - The alignment of this core feature with a core activity of Tahoe of transferring bulk data means that a substantial barrier to improved Tahoe runtime performance will be eliminated. + The alignment of this core feature with a core activity of Tahoe-LAFS of transferring bulk data means that a substantial barrier to improved Tahoe-LAFS runtime performance will be eliminated. TLS ~~~ -The Foolscap-based protocol provides *some* of Tahoe's confidentiality, integrity, and authentication properties by leveraging TLS. +The Foolscap-based protocol provides *some* of Tahoe-LAFS's confidentiality, integrity, and authentication properties by leveraging TLS. An HTTP-based protocol can make use of TLS in largely the same way to provide the same properties. Provision of these properties *is* dependant on implementers following Great Black Swamp's rules for x509 certificate validation (rather than the standard "web" rules for validation). From 89e1865f78c236b5137c3504fc4f465149bafe3e Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 2 Apr 2021 09:27:58 -0400 Subject: [PATCH 28/65] Forget about runtime performance --- docs/proposed/http-storage-node-protocol.rst | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index 83b6823e7..ad9dd30bc 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -38,11 +38,7 @@ Tahoe-LAFS pays a substantial price: The implementation is therefore the de facto standard and understanding of the protocol often relies on understanding that implementation. * The Foolscap developer community is very small. The implementation therefore advances very little and some non-trivial part of the maintenance cost falls on the Tahoe-LAFS project. -* The extensible serialization system imposes substantial overhead for the simple data structures Tahoe-LAFS exchanges. -* Foolscap encourages a "remote object" style of protocol design with involves many client-server interactions. - However, Foolscap does not implement "promise pipelining". - The result is that Foolscap encourages a protocol that requires many round-trips between client and server. -* The serialization overhead combined with the many round-trips result in Tahoe-LAFS presenting a more sluggish experience to users and taxes servers more greatly than is necessary. +* The extensible serialization system imposes substantial complexity compared to the simple data structures Tahoe-LAFS actually exchanges. HTTP ~~~~ From 939f1f840bc99947e5c545049a27ecfdafcf1cfc Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 2 Apr 2021 12:48:50 -0400 Subject: [PATCH 29/65] Fix reference to new location. --- src/allmydata/introducer/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/introducer/server.py b/src/allmydata/introducer/server.py index 6d73c4026..7f72d29f6 100644 --- a/src/allmydata/introducer/server.py +++ b/src/allmydata/introducer/server.py @@ -226,7 +226,7 @@ class IntroducerService(service.MultiService, Referenceable): # tubid will be None. Also, subscribers do not tell us which # pubkey they use; only publishers do that. tubid = rref.getRemoteTubID() or "?" - remote_address = rrefutil.stringify_remote_address(rref) + remote_address = stringify_remote_address(rref) # these three assume subscriber_info["version"]==0, but # should tolerate other versions nickname = subscriber_info.get("nickname", u"?") From dd6e0d546718a27d0ea4ff9fe03c59109113caf2 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 2 Apr 2021 12:49:01 -0400 Subject: [PATCH 30/65] Switch to decode_furl(). --- src/allmydata/introducer/common.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/allmydata/introducer/common.py b/src/allmydata/introducer/common.py index dd9bb527c..f6f70d861 100644 --- a/src/allmydata/introducer/common.py +++ b/src/allmydata/introducer/common.py @@ -12,7 +12,7 @@ if PY2: import re -from foolscap.api import SturdyRef +from foolscap.furl import decode_furl from allmydata.crypto.util import remove_prefix from allmydata.crypto import ed25519 from allmydata.util import base32, jsonbytes as json @@ -125,10 +125,10 @@ class AnnouncementDescriptor(object): self.service_name = ann_d["service-name"] self.version = ann_d.get("my-version", "") self.nickname = ann_d.get("nickname", u"") - (service_name, key_s) = index + (_, key_s) = index self.serverid = key_s furl = ann_d.get("anonymous-storage-FURL") if furl: - self.connection_hints = list(SturdyRef(furl).locationHints) + _, self.connection_hints, _ = decode_furl(furl) else: self.connection_hints = [] From c21288b3ddf40a338f53293bd7810813b3ad0f78 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 2 Apr 2021 12:49:58 -0400 Subject: [PATCH 31/65] Make it easier to read. --- src/allmydata/test/test_consumer.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/allmydata/test/test_consumer.py b/src/allmydata/test/test_consumer.py index 46b87e477..73c99d8a3 100644 --- a/src/allmydata/test/test_consumer.py +++ b/src/allmydata/test/test_consumer.py @@ -28,8 +28,8 @@ class Producer(object): If used as streaming, the test should call iterate() manually. """ - def __init__(self, consumer): - self.data = [b"abc", b"def", b"ghi"] + def __init__(self, consumer, data): + self.data = data self.consumer = consumer self.done = False @@ -58,7 +58,7 @@ class MemoryConsumerTests(TestCase): A MemoryConsumer accumulates all data sent by a streaming producer. """ consumer = MemoryConsumer() - producer = Producer(consumer) + producer = Producer(consumer, [b"abc", b"def", b"ghi"]) consumer.registerProducer(producer, True) self.assertEqual(consumer.chunks, [b"abc"]) producer.iterate() @@ -74,7 +74,7 @@ class MemoryConsumerTests(TestCase): A MemoryConsumer accumulates all data sent by a non-streaming producer. """ consumer = MemoryConsumer() - producer = Producer(consumer) + producer = Producer(consumer, [b"abc", b"def", b"ghi"]) consumer.registerProducer(producer, False) self.assertEqual(consumer.chunks, [b"abc", b"def", b"ghi"]) self.assertEqual(consumer.done, True) From 9a96fec60401edaf8ab39154e7226797e92852a5 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 2 Apr 2021 12:56:49 -0400 Subject: [PATCH 32/65] Use existing IRemoteReference implementation. --- src/allmydata/test/test_util.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/src/allmydata/test/test_util.py b/src/allmydata/test/test_util.py index 7b0b52783..2f77edf4d 100644 --- a/src/allmydata/test/test_util.py +++ b/src/allmydata/test/test_util.py @@ -28,7 +28,7 @@ from allmydata.util import yamlutil from allmydata.util import rrefutil from allmydata.util.fileutil import EncryptedTemporaryFile from allmydata.test.common_util import ReallyEqualMixin - +from .no_network import fireNow, LocalWrapper if six.PY3: long = int @@ -531,17 +531,16 @@ class JSONBytes(unittest.TestCase): self.assertEqual(json.loads(encoded, encoding="utf-8"), x) -class FakeRemoteRef(object): - """Emulate a RemoteRef.""" +class FakeGetVersion(object): + """Emulate an object with a get_version.""" def __init__(self, result): self.result = result - def callRemote(self, method): - assert method == "get_version" + def remote_get_version(self): if isinstance(self.result, Exception): - return defer.fail(self.result) - return defer.succeed(self.result) + raise self.result + return self.result class RrefUtilTests(unittest.TestCase): @@ -549,7 +548,7 @@ class RrefUtilTests(unittest.TestCase): def test_version_returned(self): """If get_version() succeeded, it is set on the rref.""" - rref = FakeRemoteRef(12345) + rref = LocalWrapper(FakeGetVersion(12345), fireNow) result = self.successResultOf( rrefutil.add_version_to_remote_reference(rref, "default") ) @@ -559,7 +558,7 @@ class RrefUtilTests(unittest.TestCase): def test_exceptions(self): """If get_version() failed, default version is set on the rref.""" for exception in (Violation(), RemoteException(ValueError())): - rref = FakeRemoteRef(exception) + rref = LocalWrapper(FakeGetVersion(exception), fireNow) result = self.successResultOf( rrefutil.add_version_to_remote_reference(rref, "Default") ) From d214fe3f16a8eeaf7fbd003ef2a1cdfe54692b88 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 2 Apr 2021 13:00:05 -0400 Subject: [PATCH 33/65] Delete unused imports. --- src/allmydata/introducer/server.py | 2 +- src/allmydata/test/test_consumer.py | 2 +- src/allmydata/test/test_util.py | 1 - src/allmydata/util/rrefutil.py | 2 +- 4 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/allmydata/introducer/server.py b/src/allmydata/introducer/server.py index 7f72d29f6..1e28f511b 100644 --- a/src/allmydata/introducer/server.py +++ b/src/allmydata/introducer/server.py @@ -29,7 +29,7 @@ from twisted.python.failure import Failure from foolscap.api import Referenceable import allmydata from allmydata import node -from allmydata.util import log, rrefutil, dictutil +from allmydata.util import log, dictutil from allmydata.util.i2p_provider import create as create_i2p_provider from allmydata.util.tor_provider import create as create_tor_provider from allmydata.introducer.interfaces import \ diff --git a/src/allmydata/test/test_consumer.py b/src/allmydata/test/test_consumer.py index 73c99d8a3..a689de462 100644 --- a/src/allmydata/test/test_consumer.py +++ b/src/allmydata/test/test_consumer.py @@ -17,7 +17,7 @@ from zope.interface import implementer from twisted.trial.unittest import TestCase from twisted.internet.interfaces import IPushProducer, IPullProducer -from allmydata.util.consumer import MemoryConsumer, download_to_data +from allmydata.util.consumer import MemoryConsumer @implementer(IPushProducer) diff --git a/src/allmydata/test/test_util.py b/src/allmydata/test/test_util.py index 2f77edf4d..9887897cf 100644 --- a/src/allmydata/test/test_util.py +++ b/src/allmydata/test/test_util.py @@ -17,7 +17,6 @@ import yaml import json from twisted.trial import unittest -from twisted.internet import defer from foolscap.api import Violation, RemoteException from allmydata.util import idlib, mathutil diff --git a/src/allmydata/util/rrefutil.py b/src/allmydata/util/rrefutil.py index 5d57a2f45..f39890ff1 100644 --- a/src/allmydata/util/rrefutil.py +++ b/src/allmydata/util/rrefutil.py @@ -10,7 +10,7 @@ from future.utils import PY2 if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 -from foolscap.api import Violation, RemoteException, SturdyRef +from foolscap.api import Violation, RemoteException def add_version_to_remote_reference(rref, default): From 9a793a93203944fec8de586450c34c7c76af6c41 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Sun, 4 Apr 2021 08:44:55 -0400 Subject: [PATCH 34/65] Turn sphinx-build's warnings into errors --- newsfragments/3666.documentation | 1 + tox.ini | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 newsfragments/3666.documentation diff --git a/newsfragments/3666.documentation b/newsfragments/3666.documentation new file mode 100644 index 000000000..3f9e34777 --- /dev/null +++ b/newsfragments/3666.documentation @@ -0,0 +1 @@ +`tox -e docs` will treat warnings about docs as errors. diff --git a/tox.ini b/tox.ini index 8908142f4..6537e1308 100644 --- a/tox.ini +++ b/tox.ini @@ -234,7 +234,7 @@ deps = # normal install is not needed for docs, and slows things down skip_install = True commands = - sphinx-build -b html -d {toxinidir}/docs/_build/doctrees {toxinidir}/docs {toxinidir}/docs/_build/html + sphinx-build -W -b html -d {toxinidir}/docs/_build/doctrees {toxinidir}/docs {toxinidir}/docs/_build/html [testenv:pyinstaller] # We override this to pass --no-use-pep517 because pyinstaller (3.4, at least) From e074034e8f88db453bc0dc46e149b65e143664b9 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 5 Apr 2021 10:24:04 -0400 Subject: [PATCH 35/65] News file. --- newsfragments/3657.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3657.minor diff --git a/newsfragments/3657.minor b/newsfragments/3657.minor new file mode 100644 index 000000000..e69de29bb From b981e90de3286bc2d4244bf304a3bd777056d2aa Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 5 Apr 2021 10:51:44 -0400 Subject: [PATCH 36/65] Tahoe-LAFS now relies on a sufficiently new version of Twisted, which includes this bugfix. --- src/allmydata/test/_twisted_9607.py | 74 ----------------------------- src/allmydata/test/test_system.py | 3 +- 2 files changed, 1 insertion(+), 76 deletions(-) delete mode 100644 src/allmydata/test/_twisted_9607.py diff --git a/src/allmydata/test/_twisted_9607.py b/src/allmydata/test/_twisted_9607.py deleted file mode 100644 index c4e37ef38..000000000 --- a/src/allmydata/test/_twisted_9607.py +++ /dev/null @@ -1,74 +0,0 @@ -""" -A copy of the implementation of Twisted's ``getProcessOutputAndValue`` -with the fix for Twisted #9607 (support for stdinBytes) patched in. -""" - -from __future__ import ( - division, - absolute_import, - print_function, - unicode_literals, -) - -from io import BytesIO - -from twisted.internet import protocol, defer - - -class _EverythingGetter(protocol.ProcessProtocol, object): - - def __init__(self, deferred, stdinBytes=None): - self.deferred = deferred - self.outBuf = BytesIO() - self.errBuf = BytesIO() - self.outReceived = self.outBuf.write - self.errReceived = self.errBuf.write - self.stdinBytes = stdinBytes - - def connectionMade(self): - if self.stdinBytes is not None: - self.transport.writeToChild(0, self.stdinBytes) - # The only compelling reason not to _always_ close stdin here is - # backwards compatibility. - self.transport.closeStdin() - - def processEnded(self, reason): - out = self.outBuf.getvalue() - err = self.errBuf.getvalue() - e = reason.value - code = e.exitCode - if e.signal: - self.deferred.errback((out, err, e.signal)) - else: - self.deferred.callback((out, err, code)) - - - -def _callProtocolWithDeferred(protocol, executable, args, env, path, - reactor=None, protoArgs=()): - if reactor is None: - from twisted.internet import reactor - - d = defer.Deferred() - p = protocol(d, *protoArgs) - reactor.spawnProcess(p, executable, (executable,)+tuple(args), env, path) - return d - - - -def getProcessOutputAndValue(executable, args=(), env={}, path=None, - reactor=None, stdinBytes=None): - """Spawn a process and returns a Deferred that will be called back with - its output (from stdout and stderr) and it's exit code as (out, err, code) - If a signal is raised, the Deferred will errback with the stdout and - stderr up to that point, along with the signal, as (out, err, signalNum) - """ - return _callProtocolWithDeferred( - _EverythingGetter, - executable, - args, - env, - path, - reactor, - protoArgs=(stdinBytes,), - ) diff --git a/src/allmydata/test/test_system.py b/src/allmydata/test/test_system.py index ce575ce7a..040104b4c 100644 --- a/src/allmydata/test/test_system.py +++ b/src/allmydata/test/test_system.py @@ -50,8 +50,7 @@ from twisted.python.failure import Failure from twisted.python.filepath import ( FilePath, ) - -from ._twisted_9607 import ( +from twisted.internet.utils import ( getProcessOutputAndValue, ) From eedc8f23cffc89c320cc065a981dacaaee67344e Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 5 Apr 2021 11:00:12 -0400 Subject: [PATCH 37/65] Delete some unused code. --- src/allmydata/test/common_util.py | 28 +--------------------------- 1 file changed, 1 insertion(+), 27 deletions(-) diff --git a/src/allmydata/test/common_util.py b/src/allmydata/test/common_util.py index f62cd34cc..be00522cd 100644 --- a/src/allmydata/test/common_util.py +++ b/src/allmydata/test/common_util.py @@ -33,13 +33,6 @@ def skip_if_cannot_represent_filename(u): except UnicodeEncodeError: raise unittest.SkipTest("A non-ASCII filename could not be encoded on this platform.") -def skip_if_cannot_represent_argv(u): - precondition(isinstance(u, unicode)) - try: - u.encode(get_io_encoding()) - except UnicodeEncodeError: - raise unittest.SkipTest("A non-ASCII argv could not be encoded on this platform.") - def _getvalue(io): """ @@ -395,27 +388,8 @@ class TimezoneMixin(object): return hasattr(time, 'tzset') -try: - import win32file - import win32con - def make_readonly(path): - win32file.SetFileAttributes(path, win32con.FILE_ATTRIBUTE_READONLY) - def make_accessible(path): - win32file.SetFileAttributes(path, win32con.FILE_ATTRIBUTE_NORMAL) -except ImportError: - import stat - def _make_readonly(path): - os.chmod(path, stat.S_IREAD) - os.chmod(os.path.dirname(path), stat.S_IREAD) - def _make_accessible(path): - os.chmod(os.path.dirname(path), stat.S_IWRITE | stat.S_IEXEC | stat.S_IREAD) - os.chmod(path, stat.S_IWRITE | stat.S_IEXEC | stat.S_IREAD) - make_readonly = _make_readonly - make_accessible = _make_accessible - - __all__ = [ - "make_readonly", "make_accessible", "TestMixin", "ShouldFailMixin", + "TestMixin", "ShouldFailMixin", "StallMixin", "skip_if_cannot_represent_argv", "run_cli", "parse_cli", "DevNullDictionary", "insecurerandstr", "flip_bit", "flip_one_bit", "SignalMixin", "skip_if_cannot_represent_filename", "ReallyEqualMixin" From 80385aea8ec9a583302f68244b68490817ac7d7b Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 5 Apr 2021 11:13:56 -0400 Subject: [PATCH 38/65] Port to Python 3. --- src/allmydata/test/cli_node_api.py | 15 +++++++++++++-- src/allmydata/util/_python3.py | 1 + 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/cli_node_api.py b/src/allmydata/test/cli_node_api.py index 34d73a199..4e4173924 100644 --- a/src/allmydata/test/cli_node_api.py +++ b/src/allmydata/test/cli_node_api.py @@ -1,3 +1,14 @@ +""" +Ported to Python 3. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 __all__ = [ "CLINodeAPI", @@ -81,7 +92,7 @@ class _ProcessProtocolAdapter(ProcessProtocol, object): self._fds = fds def connectionMade(self): - for proto in self._fds.values(): + for proto in list(self._fds.values()): proto.makeConnection(self.transport) def childDataReceived(self, childFD, data): @@ -94,7 +105,7 @@ class _ProcessProtocolAdapter(ProcessProtocol, object): def processEnded(self, reason): notified = set() - for proto in self._fds.values(): + for proto in list(self._fds.values()): if proto not in notified: proto.connectionLost(reason) notified.add(proto) diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 2f6857cad..1d31725a6 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -98,6 +98,7 @@ PORTED_MODULES = [ "allmydata.storage.shares", "allmydata.test", "allmydata.test.cli", + "allmydata.test.cli_node_api", "allmydata.test.no_network", "allmydata.test.matchers", "allmydata.test.mutable", From 84e32882b425786b14590db9e4d5f8900bccd106 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 5 Apr 2021 11:07:55 -0400 Subject: [PATCH 39/65] These don't belong in tests module. --- {src/allmydata/test => misc/checkers}/check_load.py | 0 {src/allmydata/test => misc/checkers}/check_memory.py | 0 {src/allmydata/test => misc/checkers}/check_speed.py | 0 src/allmydata/test/test_python2_regressions.py | 1 - 4 files changed, 1 deletion(-) rename {src/allmydata/test => misc/checkers}/check_load.py (100%) rename {src/allmydata/test => misc/checkers}/check_memory.py (100%) rename {src/allmydata/test => misc/checkers}/check_speed.py (100%) diff --git a/src/allmydata/test/check_load.py b/misc/checkers/check_load.py similarity index 100% rename from src/allmydata/test/check_load.py rename to misc/checkers/check_load.py diff --git a/src/allmydata/test/check_memory.py b/misc/checkers/check_memory.py similarity index 100% rename from src/allmydata/test/check_memory.py rename to misc/checkers/check_memory.py diff --git a/src/allmydata/test/check_speed.py b/misc/checkers/check_speed.py similarity index 100% rename from src/allmydata/test/check_speed.py rename to misc/checkers/check_speed.py diff --git a/src/allmydata/test/test_python2_regressions.py b/src/allmydata/test/test_python2_regressions.py index fc9ebe17a..59b16d011 100644 --- a/src/allmydata/test/test_python2_regressions.py +++ b/src/allmydata/test/test_python2_regressions.py @@ -15,7 +15,6 @@ from testtools.matchers import ( BLACKLIST = { "allmydata.scripts.types_", - "allmydata.test.check_load", "allmydata.test._win_subprocess", "allmydata.windows.registry", "allmydata.windows.fixups", From a37121f89ca638fa7621675653fc7401375d258e Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 5 Apr 2021 11:34:51 -0400 Subject: [PATCH 40/65] Already ported. --- src/allmydata/util/_python3.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 1d31725a6..d747e581f 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -103,6 +103,7 @@ PORTED_MODULES = [ "allmydata.test.matchers", "allmydata.test.mutable", "allmydata.test.mutable.util", + "allmydata.test.python3_tests", "allmydata.test.web", "allmydata.testing", "allmydata.testing.web", From 625a0abb025617dc502a06b721478b5ed841f975 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 5 Apr 2021 11:37:32 -0400 Subject: [PATCH 41/65] Port to Python 3. --- src/allmydata/test/common_web.py | 12 ++++++++++++ src/allmydata/util/_python3.py | 1 + 2 files changed, 13 insertions(+) diff --git a/src/allmydata/test/common_web.py b/src/allmydata/test/common_web.py index ce1670341..bd55a9fe9 100644 --- a/src/allmydata/test/common_web.py +++ b/src/allmydata/test/common_web.py @@ -1,3 +1,15 @@ +""" +Ported to Python 3. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + from six import ensure_str __all__ = [ diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index d747e581f..b6d4fc2ff 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -99,6 +99,7 @@ PORTED_MODULES = [ "allmydata.test", "allmydata.test.cli", "allmydata.test.cli_node_api", + "allmydata.test.common_web", "allmydata.test.no_network", "allmydata.test.matchers", "allmydata.test.mutable", From a367d333d943e73f1c4faeabbbb5a63a872a7225 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 5 Apr 2021 12:04:09 -0400 Subject: [PATCH 42/65] Port to Python 3. --- src/allmydata/test/common.py | 29 +++++++++++++++++++---------- src/allmydata/util/_python3.py | 1 + 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/src/allmydata/test/common.py b/src/allmydata/test/common.py index d7f00554d..2ce34bb1f 100644 --- a/src/allmydata/test/common.py +++ b/src/allmydata/test/common.py @@ -1,4 +1,15 @@ +""" +Ported to Python 3. +""" from __future__ import print_function +from __future__ import absolute_import +from __future__ import division +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 +from past.builtins import chr as byteschr, str as native_str __all__ = [ "SyncTestCase", @@ -15,8 +26,6 @@ __all__ = [ "PIPE", ] -from past.builtins import chr as byteschr, unicode - import sys import os, random, struct import six @@ -106,7 +115,7 @@ from .eliotutil import ( ) from .common_util import ShouldFailMixin # noqa: F401 -if sys.platform == "win32": +if sys.platform == "win32" and PY2: # Python 2.7 doesn't have good options for launching a process with # non-ASCII in its command line. So use this alternative that does a # better job. However, only use it on Windows because it doesn't work @@ -253,7 +262,7 @@ class UseNode(object): plugin_config = attr.ib() storage_plugin = attr.ib() basedir = attr.ib(validator=attr.validators.instance_of(FilePath)) - introducer_furl = attr.ib(validator=attr.validators.instance_of(str), + introducer_furl = attr.ib(validator=attr.validators.instance_of(native_str), converter=six.ensure_str) node_config = attr.ib(default=attr.Factory(dict)) @@ -264,7 +273,7 @@ class UseNode(object): return "\n".join( " = ".join((key, value)) for (key, value) - in config.items() + in list(config.items()) ) if self.plugin_config is None: @@ -849,17 +858,17 @@ class WebErrorMixin(object): callable=None, *args, **kwargs): # returns a Deferred with the response body if isinstance(substring, bytes): - substring = unicode(substring, "ascii") - if isinstance(response_substring, unicode): + substring = str(substring, "ascii") + if isinstance(response_substring, str): response_substring = response_substring.encode("ascii") - assert substring is None or isinstance(substring, unicode) + assert substring is None or isinstance(substring, str) assert response_substring is None or isinstance(response_substring, bytes) assert callable def _validate(f): if code is not None: self.failUnlessEqual(f.value.status, b"%d" % code, which) if substring: - code_string = unicode(f) + code_string = str(f) self.failUnless(substring in code_string, "%s: substring '%s' not in '%s'" % (which, substring, code_string)) @@ -882,7 +891,7 @@ class WebErrorMixin(object): body = yield response.content() self.assertEquals(response.code, code) if response_substring is not None: - if isinstance(response_substring, unicode): + if isinstance(response_substring, str): response_substring = response_substring.encode("utf-8") self.assertIn(response_substring, body) returnValue(body) diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index b6d4fc2ff..bef854054 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -99,6 +99,7 @@ PORTED_MODULES = [ "allmydata.test", "allmydata.test.cli", "allmydata.test.cli_node_api", + "allmydata.test.common", "allmydata.test.common_web", "allmydata.test.no_network", "allmydata.test.matchers", From a11b47785fc6b608d6d7d5d5b156a5d02e1930cb Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 5 Apr 2021 12:52:42 -0400 Subject: [PATCH 43/65] Port to Python 3. --- src/allmydata/test/storage_plugin.py | 11 ++++++++++- src/allmydata/util/_python3.py | 1 + 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/src/allmydata/test/storage_plugin.py b/src/allmydata/test/storage_plugin.py index 17ec89078..d3f1ec7c9 100644 --- a/src/allmydata/test/storage_plugin.py +++ b/src/allmydata/test/storage_plugin.py @@ -1,8 +1,17 @@ """ A storage server plugin the test suite can use to validate the functionality. -""" +Ported to Python 3. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from future.utils import native_str, native_str_to_bytes from six import ensure_str diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index bef854054..97cfd64f8 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -106,6 +106,7 @@ PORTED_MODULES = [ "allmydata.test.mutable", "allmydata.test.mutable.util", "allmydata.test.python3_tests", + "allmydata.test.storage_plugin", "allmydata.test.web", "allmydata.testing", "allmydata.testing.web", From 74e9bdd476d3ca67fca89047562b81b2fe14e197 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 5 Apr 2021 12:56:42 -0400 Subject: [PATCH 44/65] No point in having separate module. --- src/allmydata/test/cli/test_status.py | 18 +++++++++++++++++- src/allmydata/test/status.py | 15 --------------- 2 files changed, 17 insertions(+), 16 deletions(-) diff --git a/src/allmydata/test/cli/test_status.py b/src/allmydata/test/cli/test_status.py index a04939429..4488299b2 100644 --- a/src/allmydata/test/cli/test_status.py +++ b/src/allmydata/test/cli/test_status.py @@ -37,10 +37,26 @@ from allmydata.util import jsonbytes as json from ..no_network import GridTestMixin from ..common_web import do_http -from ..status import FakeStatus from .common import CLITestMixin +class FakeStatus(object): + def __init__(self): + self.status = [] + + def setServiceParent(self, p): + pass + + def get_status(self): + return self.status + + def get_storage_index(self): + return None + + def get_size(self): + return None + + class ProgressBar(unittest.TestCase): def test_ascii0(self): diff --git a/src/allmydata/test/status.py b/src/allmydata/test/status.py index 44f2123f9..8b1378917 100644 --- a/src/allmydata/test/status.py +++ b/src/allmydata/test/status.py @@ -1,16 +1 @@ -class FakeStatus(object): - def __init__(self): - self.status = [] - - def setServiceParent(self, p): - pass - - def get_status(self): - return self.status - - def get_storage_index(self): - return None - - def get_size(self): - return None From 3bc9b0d5440f4b08506ba7723f3d7aff4da8a2dc Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 5 Apr 2021 13:00:27 -0400 Subject: [PATCH 45/65] Port to Python 3. --- src/allmydata/test/web/matchers.py | 12 ++++++++++++ src/allmydata/util/_python3.py | 1 + 2 files changed, 13 insertions(+) diff --git a/src/allmydata/test/web/matchers.py b/src/allmydata/test/web/matchers.py index 99c91ef5c..f764da79d 100644 --- a/src/allmydata/test/web/matchers.py +++ b/src/allmydata/test/web/matchers.py @@ -1,3 +1,15 @@ +""" +Ported to Python 3. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + import attr from testtools.matchers import Mismatch diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 97cfd64f8..03eafdea5 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -108,6 +108,7 @@ PORTED_MODULES = [ "allmydata.test.python3_tests", "allmydata.test.storage_plugin", "allmydata.test.web", + "allmydata.test.web.matchers", "allmydata.testing", "allmydata.testing.web", "allmydata.unknown", From 57aa798814bcce22a7651cbdeb8217800db6571a Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 5 Apr 2021 13:01:23 -0400 Subject: [PATCH 46/65] Delete another item that was deleted. --- src/allmydata/test/common_util.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/allmydata/test/common_util.py b/src/allmydata/test/common_util.py index be00522cd..07c4a0ed7 100644 --- a/src/allmydata/test/common_util.py +++ b/src/allmydata/test/common_util.py @@ -389,8 +389,7 @@ class TimezoneMixin(object): __all__ = [ - "TestMixin", "ShouldFailMixin", - "StallMixin", "skip_if_cannot_represent_argv", "run_cli", "parse_cli", + "TestMixin", "ShouldFailMixin", "StallMixin", "run_cli", "parse_cli", "DevNullDictionary", "insecurerandstr", "flip_bit", "flip_one_bit", "SignalMixin", "skip_if_cannot_represent_filename", "ReallyEqualMixin" ] From 7b5cb13417cab0ff70d28314ad392fe437ca2eb5 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 5 Apr 2021 13:02:12 -0400 Subject: [PATCH 47/65] Flake fix. --- src/allmydata/test/common_util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/common_util.py b/src/allmydata/test/common_util.py index 07c4a0ed7..91f0b0f78 100644 --- a/src/allmydata/test/common_util.py +++ b/src/allmydata/test/common_util.py @@ -20,7 +20,7 @@ from twisted.trial import unittest from ..util.assertutil import precondition from ..scripts import runner -from allmydata.util.encodingutil import unicode_platform, get_filesystem_encoding, get_io_encoding, argv_type, unicode_to_argv +from allmydata.util.encodingutil import unicode_platform, get_filesystem_encoding, argv_type, unicode_to_argv def skip_if_cannot_represent_filename(u): From 65fd5a4912550b6da4e6d0e07cdc3aa131e7a7ef Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 5 Apr 2021 14:52:31 -0400 Subject: [PATCH 48/65] Add a note about adding Windows to GitHub Actions test matrix --- .github/workflows/ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dcbab9461..e029104b3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -28,6 +28,9 @@ jobs: # Do not run coverage tests with Python 3.6 on Windows for # now. They will fail. Dealing with them separately would # be simpler. + # + # XXX: https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3669 + # should track the effort to add Windows to the test matrix. - python-version: 3.6 os: windows-latest From 1cddae4133b67eac6e3955184c41cb251d1ea60c Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 5 Apr 2021 18:34:33 -0400 Subject: [PATCH 49/65] Add newsfragment --- newsfragments/3669.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3669.minor diff --git a/newsfragments/3669.minor b/newsfragments/3669.minor new file mode 100644 index 000000000..e69de29bb From 5f7c6e4552c75f9989c87423803137c387733649 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 5 Apr 2021 18:34:48 -0400 Subject: [PATCH 50/65] Remove Windows exclusion --- .github/workflows/ci.yml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e029104b3..c45ceaa63 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,15 +24,6 @@ jobs: python-version: - 2.7 - 3.6 - exclude: - # Do not run coverage tests with Python 3.6 on Windows for - # now. They will fail. Dealing with them separately would - # be simpler. - # - # XXX: https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3669 - # should track the effort to add Windows to the test matrix. - - python-version: 3.6 - os: windows-latest steps: From d9446f9f06e8efdddce2774e979eab5b2b685905 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 5 Apr 2021 18:57:47 -0400 Subject: [PATCH 51/65] Remove deprecated `U` mode from open() call Under the right conditions (with newer Python 3.x versions), we will see this warning: setup.py:360: DeprecationWarning: 'U' mode is deprecated `U` is for `universal newline mode`. Docs for open() says this: 'U' mode is deprecated and will raise an exception in future versions of Python. It has no effect in Python 3. Use newline to control universal newlines mode. Off it goes. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index df770fadc..308ac0334 100644 --- a/setup.py +++ b/setup.py @@ -357,7 +357,7 @@ if version: setup(name="tahoe-lafs", # also set in __init__.py description='secure, decentralized, fault-tolerant file store', - long_description=open('README.rst', 'rU').read(), + long_description=open('README.rst', 'r').read(), author='the Tahoe-LAFS project', author_email='tahoe-dev@tahoe-lafs.org', url='https://tahoe-lafs.org/', From ae6e1e9e2fde6d8029e7c7e02d3973e345708254 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 5 Apr 2021 19:11:58 -0400 Subject: [PATCH 52/65] Use io.open() instead of builtin open() Windows does not like when we open README.rst using builtin open(): Traceback (most recent call last): File "setup.py", line 360, in long_description=open('README.rst', 'rU').read(), File "c:\hostedtoolcache\windows\python\3.6.8\x64\lib\encodings\cp1252.py", line 23, in decode return codecs.charmap_decode(input,self.errors,decoding_table)[0] UnicodeDecodeError: 'charmap' codec can't decode byte 0x9d in position 1720: character maps to --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 308ac0334..df917eb40 100644 --- a/setup.py +++ b/setup.py @@ -11,6 +11,7 @@ import sys # See the docs/about.rst file for licensing information. import os, subprocess, re +from io import open basedir = os.path.dirname(os.path.abspath(__file__)) @@ -357,7 +358,7 @@ if version: setup(name="tahoe-lafs", # also set in __init__.py description='secure, decentralized, fault-tolerant file store', - long_description=open('README.rst', 'r').read(), + long_description=open('README.rst', 'r', encoding='utf-8').read(), author='the Tahoe-LAFS project', author_email='tahoe-dev@tahoe-lafs.org', url='https://tahoe-lafs.org/', From 8056b43df6ac7a247f9abefb6d6eed15e3af8c81 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 6 Apr 2021 09:21:14 -0400 Subject: [PATCH 53/65] News file. --- newsfragments/3667.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3667.minor diff --git a/newsfragments/3667.minor b/newsfragments/3667.minor new file mode 100644 index 000000000..e69de29bb From 3841662ee6a4a796277b3481ea859b3d07255c90 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 6 Apr 2021 09:23:31 -0400 Subject: [PATCH 54/65] Fix tests on Python 3. --- src/allmydata/test/common.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/common.py b/src/allmydata/test/common.py index 2ce34bb1f..d874d07ae 100644 --- a/src/allmydata/test/common.py +++ b/src/allmydata/test/common.py @@ -6,10 +6,10 @@ from __future__ import absolute_import from __future__ import division from __future__ import unicode_literals -from future.utils import PY2 +from future.utils import PY2, native_str if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 -from past.builtins import chr as byteschr, str as native_str +from past.builtins import chr as byteschr __all__ = [ "SyncTestCase", From 7ff5846b854280613fce152fe0ed0a704fb85159 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 6 Apr 2021 09:41:33 -0400 Subject: [PATCH 55/65] New towncrier has different command-line option. --- tox.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index 8908142f4..9c04ed56c 100644 --- a/tox.ini +++ b/tox.ini @@ -112,8 +112,8 @@ commands = # If towncrier.check fails, you forgot to add a towncrier news # fragment explaining the change in this branch. Create one at # `newsfragments/.` with some text for the news - # file. See pyproject.toml for legal values. - python -m towncrier.check --pyproject towncrier.pyproject.toml + # file. See towncrier.pyproject.toml for legal values. + python -m towncrier.check --config towncrier.pyproject.toml [testenv:typechecks] From bb3e80c1468ff9fc87e22cadfd8a2cd30e1a8897 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 6 Apr 2021 09:42:01 -0400 Subject: [PATCH 56/65] News file. --- newsfragments/3670.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3670.minor diff --git a/newsfragments/3670.minor b/newsfragments/3670.minor new file mode 100644 index 000000000..e69de29bb From 7f4a99306bd08d0459e00fcf2169e0ac3b744199 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 6 Apr 2021 10:04:18 -0400 Subject: [PATCH 57/65] No need to port to Python 3. --- src/allmydata/test/_win_subprocess.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/allmydata/test/_win_subprocess.py b/src/allmydata/test/_win_subprocess.py index fe6960c73..2c2cb60b4 100644 --- a/src/allmydata/test/_win_subprocess.py +++ b/src/allmydata/test/_win_subprocess.py @@ -1,3 +1,13 @@ +""" +This module is only necessary on Python 2. Once Python 2 code is dropped, it +can be deleted. +""" + +from future.utils import PY3 +if PY3: + raise RuntimeError("Just use subprocess.Popen") + + # -*- coding: utf-8 -*- ## Copyright (C) 2021 Valentin Lab From abe3fbc2e52dda4b6dfa31a298602195a09a2d64 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 6 Apr 2021 10:22:30 -0400 Subject: [PATCH 58/65] Empty file. --- src/allmydata/test/status.py | 1 - 1 file changed, 1 deletion(-) delete mode 100644 src/allmydata/test/status.py diff --git a/src/allmydata/test/status.py b/src/allmydata/test/status.py deleted file mode 100644 index 8b1378917..000000000 --- a/src/allmydata/test/status.py +++ /dev/null @@ -1 +0,0 @@ - From ae7680759dbff6a86b25c76a22e5e7fd3658caea Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 6 Apr 2021 10:22:47 -0400 Subject: [PATCH 59/65] Another checker script. --- {src/allmydata/test => misc/checkers}/check_grid.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename {src/allmydata/test => misc/checkers}/check_grid.py (100%) diff --git a/src/allmydata/test/check_grid.py b/misc/checkers/check_grid.py similarity index 100% rename from src/allmydata/test/check_grid.py rename to misc/checkers/check_grid.py From 044c79c4db9c5b5885cdb675fb39c76aa4bfd650 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 6 Apr 2021 10:29:29 -0400 Subject: [PATCH 60/65] Port to Python 3. --- src/allmydata/test/eliotutil.py | 9 +++++++++ src/allmydata/util/_python3.py | 1 + 2 files changed, 10 insertions(+) diff --git a/src/allmydata/test/eliotutil.py b/src/allmydata/test/eliotutil.py index 63c24f08a..c2359f132 100644 --- a/src/allmydata/test/eliotutil.py +++ b/src/allmydata/test/eliotutil.py @@ -1,12 +1,21 @@ """ Tools aimed at the interaction between tests and Eliot. + +Ported to Python 3. """ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals # Python 2 compatibility # Can't use `builtins.str` because it's not JSON encodable: # `exceptions.TypeError: is not JSON-encodeable` from past.builtins import unicode as str from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, max, min # noqa: F401 + from six import ensure_text __all__ = [ diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 03eafdea5..d17cbe6d8 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -101,6 +101,7 @@ PORTED_MODULES = [ "allmydata.test.cli_node_api", "allmydata.test.common", "allmydata.test.common_web", + "allmydata.test.eliotutil", "allmydata.test.no_network", "allmydata.test.matchers", "allmydata.test.mutable", From bb84442f4e939189cab1ea7142f5f3fa223f99ba Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 6 Apr 2021 10:29:46 -0400 Subject: [PATCH 61/65] News file. --- newsfragments/3671.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3671.minor diff --git a/newsfragments/3671.minor b/newsfragments/3671.minor new file mode 100644 index 000000000..e69de29bb From 2257f89d39aa9eeff0bbbfdaebad7d31c7bedfe6 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 6 Apr 2021 10:45:44 -0400 Subject: [PATCH 62/65] More semantically robust test. --- src/allmydata/test/web/test_web.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/allmydata/test/web/test_web.py b/src/allmydata/test/web/test_web.py index 6b25305c6..e73cc12f8 100644 --- a/src/allmydata/test/web/test_web.py +++ b/src/allmydata/test/web/test_web.py @@ -1394,8 +1394,8 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi def _got(res_and_status_and_headers): (res, status, headers) = res_and_status_and_headers self.failUnlessReallyEqual(res, "") - self.failUnlessReallyEqual(headers.getRawHeaders("content-length")[0], - str(len(self.BAR_CONTENTS))) + self.failUnlessReallyEqual(int(headers.getRawHeaders("content-length")[0]), + len(self.BAR_CONTENTS)) self.failUnlessReallyEqual(headers.getRawHeaders("content-type"), ["text/plain"]) d.addCallback(_got) @@ -3015,8 +3015,8 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi def _got_headers(res_and_status_and_headers): (res, status, headers) = res_and_status_and_headers self.failUnlessReallyEqual(res, "") - self.failUnlessReallyEqual(headers.getRawHeaders("content-length")[0], - str(len(NEW2_CONTENTS))) + self.failUnlessReallyEqual(int(headers.getRawHeaders("content-length")[0]), + len(NEW2_CONTENTS)) self.failUnlessReallyEqual(headers.getRawHeaders("content-type"), ["text/plain"]) d.addCallback(_got_headers) From 3429f8bf030b8f2713fcabb13cf49fed8e079279 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 6 Apr 2021 10:45:58 -0400 Subject: [PATCH 63/65] Port to Python 3. --- src/allmydata/test/common_util.py | 21 ++++++++++++++------- src/allmydata/util/_python3.py | 1 + 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/src/allmydata/test/common_util.py b/src/allmydata/test/common_util.py index 91f0b0f78..f4cc4a074 100644 --- a/src/allmydata/test/common_util.py +++ b/src/allmydata/test/common_util.py @@ -1,8 +1,15 @@ +""" +Ported to Python 3. +""" from __future__ import print_function +from __future__ import absolute_import +from __future__ import division +from __future__ import unicode_literals from future.utils import PY2, bchr, binary_type from future.builtins import str as future_str -from past.builtins import unicode +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, dict, list, object, range, str, max, min # noqa: F401 import os import time @@ -24,7 +31,7 @@ from allmydata.util.encodingutil import unicode_platform, get_filesystem_encodin def skip_if_cannot_represent_filename(u): - precondition(isinstance(u, unicode)) + precondition(isinstance(u, str)) enc = get_filesystem_encoding() if not unicode_platform(): @@ -44,7 +51,7 @@ def _getvalue(io): def maybe_unicode_to_argv(o): """Convert object to argv form if necessary.""" - if isinstance(o, unicode): + if isinstance(o, str): return unicode_to_argv(o) return o @@ -181,7 +188,7 @@ class DevNullDictionary(dict): return def insecurerandstr(n): - return b''.join(map(bchr, map(randrange, [0]*n, [256]*n))) + return b''.join(map(bchr, list(map(randrange, [0]*n, [256]*n)))) def flip_bit(good, which): """Flip the low-order bit of good[which].""" @@ -211,9 +218,9 @@ class ReallyEqualMixin(object): # type. They're equal, and _logically_ the same type, but have # different types in practice. if a.__class__ == future_str: - a = unicode(a) + a = str(a) if b.__class__ == future_str: - b = unicode(b) + b = str(b) self.assertEqual(type(a), type(b), "a :: %r (%s), b :: %r (%s), %r" % (a, type(a), b, type(b), msg)) @@ -297,7 +304,7 @@ class ShouldFailMixin(object): of the message wrapped by this Failure, or the test will fail. """ - assert substring is None or isinstance(substring, (bytes, unicode)) + assert substring is None or isinstance(substring, (bytes, str)) d = defer.maybeDeferred(callable, *args, **kwargs) def done(res): if isinstance(res, failure.Failure): diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index d17cbe6d8..4676f6955 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -100,6 +100,7 @@ PORTED_MODULES = [ "allmydata.test.cli", "allmydata.test.cli_node_api", "allmydata.test.common", + "allmydata.test.common_util", "allmydata.test.common_web", "allmydata.test.eliotutil", "allmydata.test.no_network", From 315bb672d11ca77de93edef9b07d82c4a6bc8405 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 6 Apr 2021 10:49:00 -0400 Subject: [PATCH 64/65] Port to Python 3. --- src/allmydata/test/web/common.py | 11 +++++++++++ src/allmydata/util/_python3.py | 1 + 2 files changed, 12 insertions(+) diff --git a/src/allmydata/test/web/common.py b/src/allmydata/test/web/common.py index 00a40e3c5..43a13a902 100644 --- a/src/allmydata/test/web/common.py +++ b/src/allmydata/test/web/common.py @@ -1,3 +1,14 @@ +""" +Ported to Python 3. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import re diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 4676f6955..5d70d1e73 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -110,6 +110,7 @@ PORTED_MODULES = [ "allmydata.test.python3_tests", "allmydata.test.storage_plugin", "allmydata.test.web", + "allmydata.test.web.common", "allmydata.test.web.matchers", "allmydata.testing", "allmydata.testing.web", From ddcca38f3148abb020e0f4d8139dac9bc3e9e966 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 6 Apr 2021 10:51:14 -0400 Subject: [PATCH 65/65] Port to Python 3. --- src/allmydata/test/strategies.py | 10 ++++++++++ src/allmydata/util/_python3.py | 1 + 2 files changed, 11 insertions(+) diff --git a/src/allmydata/test/strategies.py b/src/allmydata/test/strategies.py index 553b2c226..c0f558ef6 100644 --- a/src/allmydata/test/strategies.py +++ b/src/allmydata/test/strategies.py @@ -1,6 +1,16 @@ """ Hypothesis strategies use for testing Tahoe-LAFS. + +Ported to Python 3. """ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from hypothesis.strategies import ( one_of, diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 5d70d1e73..3f705fe09 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -109,6 +109,7 @@ PORTED_MODULES = [ "allmydata.test.mutable.util", "allmydata.test.python3_tests", "allmydata.test.storage_plugin", + "allmydata.test.strategies", "allmydata.test.web", "allmydata.test.web.common", "allmydata.test.web.matchers",