From ee9d9d79848cfcc5e4e49e094db07c25082f9b64 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 11 Sep 2020 15:38:04 -0400 Subject: [PATCH 001/154] Add mypy checks as separate tox environment. --- mypy.ini | 2 ++ tox.ini | 7 +++++++ 2 files changed, 9 insertions(+) create mode 100644 mypy.ini diff --git a/mypy.ini b/mypy.ini new file mode 100644 index 000000000..976ba0294 --- /dev/null +++ b/mypy.ini @@ -0,0 +1,2 @@ +[mypy] +ignore_missing_imports = True diff --git a/tox.ini b/tox.ini index 597270e3a..f5c8f3f42 100644 --- a/tox.ini +++ b/tox.ini @@ -108,6 +108,13 @@ commands = # file. See pyproject.toml for legal values. python -m towncrier.check --pyproject towncrier.pyproject.toml + +[testenv:typechecks] +skip_install = True +deps = mypy +commands = mypy src + + [testenv:draftnews] passenv = TAHOE_LAFS_* PIP_* SUBUNITREPORTER_* USERPROFILE HOMEDRIVE HOMEPATH # see comment in [testenv] about "certifi" From ab54585558f32c92aa9e786df19c42d577e15a44 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 18 Sep 2020 15:49:21 -0400 Subject: [PATCH 002/154] Incorporate mypy-zope to support zope interfaces. --- mypy.ini | 1 + tox.ini | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/mypy.ini b/mypy.ini index 976ba0294..01cbb57a8 100644 --- a/mypy.ini +++ b/mypy.ini @@ -1,2 +1,3 @@ [mypy] ignore_missing_imports = True +plugins=mypy_zope:plugin diff --git a/tox.ini b/tox.ini index f5c8f3f42..1c232b4d7 100644 --- a/tox.ini +++ b/tox.ini @@ -111,7 +111,9 @@ commands = [testenv:typechecks] skip_install = True -deps = mypy +deps = + mypy + mypy-zope commands = mypy src From 6d2d82d7b7342be42d0392ccac0af3ed8882f41c Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Tue, 13 Oct 2020 21:06:07 -0400 Subject: [PATCH 003/154] Use pre-release versions of foolscap and mypy-zope with intended support for RemoteInterface subclasses. --- tox.ini | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 1c232b4d7..a13e840a4 100644 --- a/tox.ini +++ b/tox.ini @@ -113,7 +113,8 @@ commands = skip_install = True deps = mypy - mypy-zope + git+https://github.com/jaraco/mypy-zope@bugfix/21-InterfaceClass-subclass + git+https://github.com/jaraco/foolscap@bugfix/75-use-metaclass commands = mypy src From 4b559ffc3332222c99a86b0f75ea05a01a845321 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 31 Oct 2020 16:06:52 -0400 Subject: [PATCH 004/154] Add typechecks to tox run --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index a13e840a4..0cc0c4ac2 100644 --- a/tox.ini +++ b/tox.ini @@ -7,7 +7,7 @@ twisted = 1 [tox] -envlist = codechecks,py27,py36,pypy27 +envlist = typechecks,codechecks,py27,py36,pypy27 minversion = 2.4 [testenv] From f2ffa78198e756a2337a66463241127c38318509 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 20 Nov 2020 12:32:11 -0500 Subject: [PATCH 005/154] Define type of PollMixin._poll_should_ignore_these_errors --- src/allmydata/util/pollmixin.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/allmydata/util/pollmixin.py b/src/allmydata/util/pollmixin.py index 5d1716853..582bafe86 100644 --- a/src/allmydata/util/pollmixin.py +++ b/src/allmydata/util/pollmixin.py @@ -14,6 +14,12 @@ if PY2: from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import time + +try: + from typing import List +except ImportError: + pass + from twisted.internet import task class TimeoutError(Exception): @@ -23,7 +29,7 @@ class PollComplete(Exception): pass class PollMixin(object): - _poll_should_ignore_these_errors = [] + _poll_should_ignore_these_errors = [] # type: List[Exception] def poll(self, check_f, pollinterval=0.01, timeout=1000): # Return a Deferred, then call check_f periodically until it returns From ce3b775944fdd9070f57de1230c4b2f1c1b8a5b7 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 20 Nov 2020 12:33:41 -0500 Subject: [PATCH 006/154] Suppress typing error in test_python3 --- src/allmydata/test/test_python3.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/test_python3.py b/src/allmydata/test/test_python3.py index 80242f8a2..c1f0e83d6 100644 --- a/src/allmydata/test/test_python3.py +++ b/src/allmydata/test/test_python3.py @@ -44,7 +44,7 @@ class Python3PortingEffortTests(SynchronousTestCase): ), ), ) - test_finished_porting.todo = native_str( + test_finished_porting.todo = native_str( # type: ignore "https://tahoe-lafs.org/trac/tahoe-lafs/milestone/Support%20Python%203 should be completed", ) From d1ea36781a651ece31807c32f366ce6c596b8e70 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 20 Nov 2020 13:12:52 -0500 Subject: [PATCH 007/154] Add type declarations to check_load. --- src/allmydata/test/check_load.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/allmydata/test/check_load.py b/src/allmydata/test/check_load.py index 4058ddf77..21576ea3a 100644 --- a/src/allmydata/test/check_load.py +++ b/src/allmydata/test/check_load.py @@ -37,6 +37,11 @@ a mean of 10kB and a max of 100MB, so filesize=min(int(1.0/random(.0002)),1e8) import os, sys, httplib, binascii import urllib, json, random, time, urlparse +try: + from typing import Dict +except ImportError: + pass + # Python 2 compatibility from future.utils import PY2 if PY2: @@ -49,13 +54,13 @@ if sys.argv[1] == "--stats": DELAY = 10 MAXSAMPLES = 6 totals = [] - last_stats = {} + last_stats = {} # type: Dict[str, float] while True: - stats = {} + stats = {} # type: Dict[str, float] for sf in statsfiles: for line in open(sf, "r").readlines(): - name, value = line.split(":") - value = int(value.strip()) + name, str_value = line.split(":") + value = int(str_value.strip()) if name not in stats: stats[name] = 0 stats[name] += float(value) From 8da82e9ed55d363fb963b95e9b96fac79e6c1bb8 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 20 Nov 2020 13:47:23 -0500 Subject: [PATCH 008/154] Add workaround for Shoobx/mypy-zope#26. --- src/allmydata/web/private.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/allmydata/web/private.py b/src/allmydata/web/private.py index fea058405..a86c869c4 100644 --- a/src/allmydata/web/private.py +++ b/src/allmydata/web/private.py @@ -61,7 +61,11 @@ class IToken(ICredentials): pass -@implementer(IToken) +# Shoobx/mypy-zope#26 +_itoken_impl = implementer(IToken) + + +@_itoken_impl @attr.s class Token(object): proposed_token = attr.ib(type=bytes) From 25cce8b77ebbf6d195f938d39e0c00aa9e6262b9 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Mon, 23 Nov 2020 13:57:37 -0500 Subject: [PATCH 009/154] Suppress typing errors in fileutil, crawler, fixups. --- src/allmydata/storage/crawler.py | 2 +- src/allmydata/util/fileutil.py | 2 +- src/allmydata/windows/fixups.py | 7 ++++++- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/allmydata/storage/crawler.py b/src/allmydata/storage/crawler.py index 24042c38b..f13f7cb99 100644 --- a/src/allmydata/storage/crawler.py +++ b/src/allmydata/storage/crawler.py @@ -19,7 +19,7 @@ import os, time, struct try: import cPickle as pickle except ImportError: - import pickle + import pickle # type: ignore from twisted.internet import reactor from twisted.application import service from allmydata.storage.common import si_b2a diff --git a/src/allmydata/util/fileutil.py b/src/allmydata/util/fileutil.py index ea16c0d6a..e40e06180 100644 --- a/src/allmydata/util/fileutil.py +++ b/src/allmydata/util/fileutil.py @@ -311,7 +311,7 @@ def precondition_abspath(path): _getfullpathname = None try: - from nt import _getfullpathname + from nt import _getfullpathname # type: ignore except ImportError: pass diff --git a/src/allmydata/windows/fixups.py b/src/allmydata/windows/fixups.py index e7f045b95..c5ba3bb57 100644 --- a/src/allmydata/windows/fixups.py +++ b/src/allmydata/windows/fixups.py @@ -217,7 +217,12 @@ def initialize(): # Instead it "mangles" or escapes them using \x7F as an escape character, which we # unescape here. def unmangle(s): - return re.sub(u'\\x7F[0-9a-fA-F]*\\;', lambda m: unichr(int(m.group(0)[1:-1], 16)), s) + return re.sub( + u'\\x7F[0-9a-fA-F]*\\;', + # type ignored for 'unichr' + lambda m: unichr(int(m.group(0)[1:-1], 16)), # type: ignore + s, + ) try: argv = [unmangle(argv_unicode[i]).encode('utf-8') for i in xrange(0, argc.value)] From df31d7db5b23bbc1e12d83f44bf23c9cce316b25 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Mon, 23 Nov 2020 14:05:33 -0500 Subject: [PATCH 010/154] Suppress type error in Node.GENERATED_FILES, apparently unused. --- src/allmydata/node.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/node.py b/src/allmydata/node.py index 0dcd900aa..7622d5bc3 100644 --- a/src/allmydata/node.py +++ b/src/allmydata/node.py @@ -713,7 +713,7 @@ class Node(service.MultiService): """ NODETYPE = "unknown NODETYPE" CERTFILE = "node.pem" - GENERATED_FILES = [] + GENERATED_FILES = [] # type: ignore def __init__(self, config, main_tub, control_tub, i2p_provider, tor_provider): """ From dec6f6d64705b0f103dba3436598f42e96df365c Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Mon, 23 Nov 2020 14:08:23 -0500 Subject: [PATCH 011/154] Remove Interface subclass, as IURI is an interface. Fixes mypy error. --- src/allmydata/interfaces.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/interfaces.py b/src/allmydata/interfaces.py index 49dcf7646..537f6d655 100644 --- a/src/allmydata/interfaces.py +++ b/src/allmydata/interfaces.py @@ -682,7 +682,7 @@ class IURI(Interface): passing into init_from_string.""" -class IVerifierURI(Interface, IURI): +class IVerifierURI(IURI): def init_from_string(uri): """Accept a string (as created by my to_string() method) and populate this instance with its data. I am not normally called directly, From 5f40c562ebb0754c3eabfb0b1dddaeb6d9e4e3a6 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Mon, 23 Nov 2020 14:09:46 -0500 Subject: [PATCH 012/154] Remove self arguments to IProgress, which mypy caught as improper. --- src/allmydata/interfaces.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/interfaces.py b/src/allmydata/interfaces.py index 537f6d655..95b1fdf63 100644 --- a/src/allmydata/interfaces.py +++ b/src/allmydata/interfaces.py @@ -749,7 +749,7 @@ class IProgress(Interface): "Current amount of progress (in percentage)" ) - def set_progress(self, value): + def set_progress(value): """ Sets the current amount of progress. @@ -757,7 +757,7 @@ class IProgress(Interface): set_progress_total. """ - def set_progress_total(self, value): + def set_progress_total(value): """ Sets the total amount of expected progress From adf06889181c41c20db56aece8537a46945ae3ea Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Mon, 23 Nov 2020 14:15:39 -0500 Subject: [PATCH 013/154] Add a non-implementation of encode_proposal to satisfy interface. --- src/allmydata/codec.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/allmydata/codec.py b/src/allmydata/codec.py index a4baab4b6..19345959e 100644 --- a/src/allmydata/codec.py +++ b/src/allmydata/codec.py @@ -57,6 +57,10 @@ class CRSEncoder(object): return defer.succeed((shares, desired_share_ids)) + def encode_proposal(self, data, desired_share_ids=None): + raise NotImplementedError() + + @implementer(ICodecDecoder) class CRSDecoder(object): From 4998c4693fdfd900d0659000888e0e0a59346b8c Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 29 Nov 2020 13:46:52 -0500 Subject: [PATCH 014/154] Ignore type checks on Referenceable objects. Ref warner/foolscap#78. --- src/allmydata/immutable/offloaded.py | 4 ++-- src/allmydata/immutable/upload.py | 2 +- src/allmydata/storage/immutable.py | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/allmydata/immutable/offloaded.py b/src/allmydata/immutable/offloaded.py index d574b980d..4e18ad216 100644 --- a/src/allmydata/immutable/offloaded.py +++ b/src/allmydata/immutable/offloaded.py @@ -141,7 +141,7 @@ class CHKCheckerAndUEBFetcher(object): @implementer(interfaces.RICHKUploadHelper) -class CHKUploadHelper(Referenceable, upload.CHKUploader): +class CHKUploadHelper(Referenceable, upload.CHKUploader): # type: ignore # warner/foolscap#78 """I am the helper-server -side counterpart to AssistedUploader. I handle peer selection, encoding, and share pushing. I read ciphertext from the remote AssistedUploader. @@ -502,7 +502,7 @@ class LocalCiphertextReader(AskUntilSuccessMixin): @implementer(interfaces.RIHelper, interfaces.IStatsProducer) -class Helper(Referenceable): +class Helper(Referenceable): # type: ignore # warner/foolscap#78 """ :ivar dict[bytes, CHKUploadHelper] _active_uploads: For any uploads which have been started but not finished, a mapping from storage index to the diff --git a/src/allmydata/immutable/upload.py b/src/allmydata/immutable/upload.py index e77cbb30b..18f818504 100644 --- a/src/allmydata/immutable/upload.py +++ b/src/allmydata/immutable/upload.py @@ -1423,7 +1423,7 @@ class LiteralUploader(object): return self._status @implementer(RIEncryptedUploadable) -class RemoteEncryptedUploadable(Referenceable): +class RemoteEncryptedUploadable(Referenceable): # type: ignore # warner/foolscap#78 def __init__(self, encrypted_uploadable, upload_status): self._eu = IEncryptedUploadable(encrypted_uploadable) diff --git a/src/allmydata/storage/immutable.py b/src/allmydata/storage/immutable.py index 778c0ddf8..4b60d79f1 100644 --- a/src/allmydata/storage/immutable.py +++ b/src/allmydata/storage/immutable.py @@ -202,7 +202,7 @@ class ShareFile(object): @implementer(RIBucketWriter) -class BucketWriter(Referenceable): +class BucketWriter(Referenceable): # type: ignore # warner/foolscap#78 def __init__(self, ss, incominghome, finalhome, max_size, lease_info, canary): self.ss = ss @@ -301,7 +301,7 @@ class BucketWriter(Referenceable): @implementer(RIBucketReader) -class BucketReader(Referenceable): +class BucketReader(Referenceable): # type: ignore # warner/foolscap#78 def __init__(self, ss, sharefname, storage_index=None, shnum=None): self.ss = ss From 50f81aa25d7ad86c5c238bc2a1d70afce25de03f Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 29 Nov 2020 13:54:39 -0500 Subject: [PATCH 015/154] Update two methods of introducer.client.IntroducerClient to match the interface definition. --- src/allmydata/introducer/client.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/allmydata/introducer/client.py b/src/allmydata/introducer/client.py index 0a6352317..62642d0af 100644 --- a/src/allmydata/introducer/client.py +++ b/src/allmydata/introducer/client.py @@ -157,15 +157,15 @@ class IntroducerClient(service.Service, Referenceable): kwargs["facility"] = "tahoe.introducer.client" return log.msg(*args, **kwargs) - def subscribe_to(self, service_name, cb, *args, **kwargs): - self._local_subscribers.append( (service_name,cb,args,kwargs) ) + def subscribe_to(self, service_name, callback, *args, **kwargs): + self._local_subscribers.append( (service_name,callback,args,kwargs) ) self._subscribed_service_names.add(service_name) self._maybe_subscribe() for index,(ann,key_s,when) in self._inbound_announcements.items(): precondition(isinstance(key_s, str), key_s) servicename = index[0] if servicename == service_name: - eventually(cb, key_s, ann, *args, **kwargs) + eventually(callback, key_s, ann, *args, **kwargs) def _maybe_subscribe(self): if not self._publisher: @@ -198,7 +198,7 @@ class IntroducerClient(service.Service, Referenceable): ann_d.update(ann) return ann_d - def publish(self, service_name, ann, signing_key): + def publish(self, service_name, ann, signing_key=None): # we increment the seqnum every time we publish something new current_seqnum, current_nonce = self._sequencer() From bc3508ce6098d65ae0407047a2e02d3a1bacfee9 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 29 Nov 2020 14:01:05 -0500 Subject: [PATCH 016/154] Ignore type checks on cmp usage (awaiting Python 3 porting) --- src/allmydata/web/status.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/web/status.py b/src/allmydata/web/status.py index 7f6020a99..de15230fd 100644 --- a/src/allmydata/web/status.py +++ b/src/allmydata/web/status.py @@ -1335,7 +1335,7 @@ class Status(MultiFormatResource): active = [s for s in self._get_all_statuses() if s.get_active()] - active.sort(lambda a, b: cmp(a.get_started(), b.get_started())) + active.sort(lambda a, b: cmp(a.get_started(), b.get_started())) # type: ignore # py2 active.reverse() return active @@ -1343,7 +1343,7 @@ class Status(MultiFormatResource): recent = [s for s in self._get_all_statuses() if not s.get_active()] - recent.sort(lambda a, b: cmp(a.get_started(), b.get_started())) + recent.sort(lambda a, b: cmp(a.get_started(), b.get_started())) # type: ignore # py2 recent.reverse() return recent From 6ba7533168eff65454376987258748f99fb1057d Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 29 Nov 2020 14:04:59 -0500 Subject: [PATCH 017/154] Ignore failure on StorageServer.slot_testv_and_readv_and_writev, the implementation of which deviates from the interface spec substantially. --- src/allmydata/storage/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py index 8a8138f26..b7df702d5 100644 --- a/src/allmydata/storage/server.py +++ b/src/allmydata/storage/server.py @@ -581,7 +581,7 @@ class StorageServer(service.MultiService, Referenceable): for share in six.viewvalues(shares): share.add_or_renew_lease(lease_info) - def slot_testv_and_readv_and_writev( + def slot_testv_and_readv_and_writev( # type: ignore # fixme self, storage_index, secrets, From dca0840c350e5b188672e7410a4995b50a1dca60 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 29 Nov 2020 14:08:33 -0500 Subject: [PATCH 018/154] Add stubs for methods demanded by the interface --- src/allmydata/uri.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/allmydata/uri.py b/src/allmydata/uri.py index 2c367cafe..e95c86d96 100644 --- a/src/allmydata/uri.py +++ b/src/allmydata/uri.py @@ -535,6 +535,12 @@ class _DirectoryBaseURI(_BaseURI): def get_storage_index(self): return self._filenode_uri.get_storage_index() + def get_readonly(self): + raise NotImplementedError() + + def is_readonly(self): + raise NotImplementedError() + @implementer(IDirectoryURI) class DirectoryURI(_DirectoryBaseURI): From cc91b7c9edf0d5fc64525b410b50da0748f1180b Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 29 Nov 2020 14:15:24 -0500 Subject: [PATCH 019/154] Declare DirectoryURIVerifier type to allow subclass to override. --- src/allmydata/uri.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/allmydata/uri.py b/src/allmydata/uri.py index e95c86d96..d7f4782cd 100644 --- a/src/allmydata/uri.py +++ b/src/allmydata/uri.py @@ -22,6 +22,11 @@ from past.builtins import unicode, long import re +try: + from typing import Type +except ImportError: + pass + from zope.interface import implementer from twisted.python.components import registerAdapter @@ -707,7 +712,7 @@ class DirectoryURIVerifier(_DirectoryBaseURI): BASE_STRING=b'URI:DIR2-Verifier:' BASE_STRING_RE=re.compile(b'^'+BASE_STRING) - INNER_URI_CLASS=SSKVerifierURI + INNER_URI_CLASS=SSKVerifierURI # type: Type[IVerifierURI] def __init__(self, filenode_uri=None): if filenode_uri: From 7e757d2ec4e201d4e9a1e03be66bd5007eb5c094 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 29 Nov 2020 14:23:59 -0500 Subject: [PATCH 020/154] As _ImmutableFileNodeBase doesn't implement the interface, move the implementer declaration to LiteralFileNode --- src/allmydata/immutable/literal.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/allmydata/immutable/literal.py b/src/allmydata/immutable/literal.py index 68db478f3..6ed5571b9 100644 --- a/src/allmydata/immutable/literal.py +++ b/src/allmydata/immutable/literal.py @@ -19,7 +19,7 @@ from twisted.protocols import basic from allmydata.interfaces import IImmutableFileNode, ICheckable from allmydata.uri import LiteralFileURI -@implementer(IImmutableFileNode, ICheckable) + class _ImmutableFileNodeBase(object): def get_write_uri(self): @@ -56,6 +56,7 @@ class _ImmutableFileNodeBase(object): return not self == other +@implementer(IImmutableFileNode, ICheckable) class LiteralFileNode(_ImmutableFileNodeBase): def __init__(self, filecap): From e9ddcf5911dd75cceea8611dd7a134d7a7eb2fb1 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 29 Nov 2020 14:26:20 -0500 Subject: [PATCH 021/154] Implement set_size as required by the interface --- src/allmydata/immutable/encode.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/allmydata/immutable/encode.py b/src/allmydata/immutable/encode.py index 9351df501..e743ce766 100644 --- a/src/allmydata/immutable/encode.py +++ b/src/allmydata/immutable/encode.py @@ -711,3 +711,6 @@ class Encoder(object): return self.uri_extension_data def get_uri_extension_hash(self): return self.uri_extension_hash + + def set_size(self, size): + raise NotImplementedError() From 1248d65778242c81e914fa6096990c3e3e68fcba Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 29 Nov 2020 14:32:34 -0500 Subject: [PATCH 022/154] Declare types for BasedirOptions. Fixes several errors. --- src/allmydata/scripts/common.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/allmydata/scripts/common.py b/src/allmydata/scripts/common.py index 34266ee72..a633da655 100644 --- a/src/allmydata/scripts/common.py +++ b/src/allmydata/scripts/common.py @@ -4,6 +4,11 @@ import os, sys, urllib, textwrap import codecs from os.path import join +try: + from typing import Optional, Sequence, List +except ImportError: + pass + # Python 2 compatibility from future.utils import PY2 if PY2: @@ -64,7 +69,7 @@ class BasedirOptions(BaseOptions): optParameters = [ ["basedir", "C", None, "Specify which Tahoe base directory should be used. [default: %s]" % quote_local_unicode_path(_default_nodedir)], - ] + ] # type: List[Sequence[Optional[str]]] def parseArgs(self, basedir=None): # This finds the node-directory option correctly even if we are in a subcommand. From e0eb63929a56e46d622fa48b1be159b09cefc031 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 29 Nov 2020 14:35:17 -0500 Subject: [PATCH 023/154] Declare type for BaseOptions.description. Fixes many type errors. --- src/allmydata/scripts/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/scripts/common.py b/src/allmydata/scripts/common.py index a633da655..6d281e07c 100644 --- a/src/allmydata/scripts/common.py +++ b/src/allmydata/scripts/common.py @@ -45,7 +45,7 @@ class BaseOptions(usage.Options): def opt_version(self): raise usage.UsageError("--version not allowed on subcommands") - description = None + description = None # type: Optional[str] description_unwrapped = None def __str__(self): From 1b92da75fa01d94a8486e1a4108c3dcce10458e8 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 29 Nov 2020 14:44:17 -0500 Subject: [PATCH 024/154] Some subclasses use ints, so just go for Any --- src/allmydata/scripts/common.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/scripts/common.py b/src/allmydata/scripts/common.py index 6d281e07c..438b4a7e2 100644 --- a/src/allmydata/scripts/common.py +++ b/src/allmydata/scripts/common.py @@ -5,7 +5,7 @@ import codecs from os.path import join try: - from typing import Optional, Sequence, List + from typing import Optional, Sequence, List, Any except ImportError: pass @@ -69,7 +69,7 @@ class BasedirOptions(BaseOptions): optParameters = [ ["basedir", "C", None, "Specify which Tahoe base directory should be used. [default: %s]" % quote_local_unicode_path(_default_nodedir)], - ] # type: List[Sequence[Optional[str]]] + ] # type: List[Sequence[Any]] def parseArgs(self, basedir=None): # This finds the node-directory option correctly even if we are in a subcommand. From c3a22966e82fd698eab1d62b5416be53b863013b Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 29 Nov 2020 14:52:45 -0500 Subject: [PATCH 025/154] Add stubs for methods demanded by IPeerSelector --- src/allmydata/immutable/upload.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/allmydata/immutable/upload.py b/src/allmydata/immutable/upload.py index 18f818504..171b71eff 100644 --- a/src/allmydata/immutable/upload.py +++ b/src/allmydata/immutable/upload.py @@ -385,6 +385,12 @@ class PeerSelector(object): ) return self.happiness_mappings + def add_peers(self, peerids=None): + raise NotImplementedError + + def confirm_share_allocation(self, peerid, shnum): + raise NotImplementedError + class _QueryStatistics(object): From af172f6bff9b5de980473a02155840e50cdbae91 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 29 Nov 2020 14:55:13 -0500 Subject: [PATCH 026/154] Repeat type declaration from parent to avoid over-constraining this type for subclasses. --- src/allmydata/scripts/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/scripts/common.py b/src/allmydata/scripts/common.py index 438b4a7e2..81511681d 100644 --- a/src/allmydata/scripts/common.py +++ b/src/allmydata/scripts/common.py @@ -106,7 +106,7 @@ class NoDefaultBasedirOptions(BasedirOptions): optParameters = [ ["basedir", "C", None, "Specify which Tahoe base directory should be used."], - ] + ] # type: List[Sequence[Any]] # This is overridden in order to ensure we get a "Wrong number of arguments." # error when more than one argument is given. From 103bec6a1579373df3238c8b5ee09eb72f79718a Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 29 Nov 2020 14:58:28 -0500 Subject: [PATCH 027/154] On MutableFileNode, accept optional 'progress' parameter as declared by the interface. --- src/allmydata/mutable/filenode.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/mutable/filenode.py b/src/allmydata/mutable/filenode.py index 5afc84dec..54e9844d6 100644 --- a/src/allmydata/mutable/filenode.py +++ b/src/allmydata/mutable/filenode.py @@ -564,7 +564,7 @@ class MutableFileNode(object): return d - def upload(self, new_contents, servermap): + def upload(self, new_contents, servermap, progress=None): """ I overwrite the contents of the best recoverable version of this mutable file with new_contents, using servermap instead of From a75454a04f61c3e96b080553fe1f7e412b1d976b Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 29 Nov 2020 15:02:13 -0500 Subject: [PATCH 028/154] Add stub for MutableFileVersion.get_servermap --- src/allmydata/mutable/filenode.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/allmydata/mutable/filenode.py b/src/allmydata/mutable/filenode.py index 54e9844d6..e9cf23fa1 100644 --- a/src/allmydata/mutable/filenode.py +++ b/src/allmydata/mutable/filenode.py @@ -1205,3 +1205,6 @@ class MutableFileVersion(object): self._servermap, mode=mode) return u.update() + + def get_servermap(self): + raise NotImplementedError From 32b77c42394c337fd2bc38f6ff2be29e5acdd001 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 29 Nov 2020 15:06:12 -0500 Subject: [PATCH 029/154] Ignore interface violation in MutableFileVersion.download_to_data --- src/allmydata/mutable/filenode.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/mutable/filenode.py b/src/allmydata/mutable/filenode.py index e9cf23fa1..4613a918b 100644 --- a/src/allmydata/mutable/filenode.py +++ b/src/allmydata/mutable/filenode.py @@ -951,7 +951,7 @@ class MutableFileVersion(object): return self._servermap.size_of_version(self._version) - def download_to_data(self, fetch_privkey=False, progress=None): + def download_to_data(self, fetch_privkey=False, progress=None): # type: ignore # fixme """ I return a Deferred that fires with the contents of this readable object as a byte string. From 646297ddc3f95b9ab88b35d32a83782c8e2f178f Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 29 Nov 2020 15:08:46 -0500 Subject: [PATCH 030/154] Add stub for LocalCiphertextReader.set_upload_status --- src/allmydata/immutable/offloaded.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/allmydata/immutable/offloaded.py b/src/allmydata/immutable/offloaded.py index 4e18ad216..53a1f911a 100644 --- a/src/allmydata/immutable/offloaded.py +++ b/src/allmydata/immutable/offloaded.py @@ -499,6 +499,8 @@ class LocalCiphertextReader(AskUntilSuccessMixin): # ??. I'm not sure if it makes sense to forward the close message. return self.call("close") + def set_upload_status(self, upload_status): + raise NotImplementedError @implementer(interfaces.RIHelper, interfaces.IStatsProducer) From 67f0be8431157f965df50bd82e3098b9be3bbe12 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 29 Nov 2020 15:13:19 -0500 Subject: [PATCH 031/154] Prefer type(None) for better compatibility. --- src/allmydata/frontends/ftpd.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/allmydata/frontends/ftpd.py b/src/allmydata/frontends/ftpd.py index 0b18df85b..af5444969 100644 --- a/src/allmydata/frontends/ftpd.py +++ b/src/allmydata/frontends/ftpd.py @@ -1,7 +1,5 @@ from six import ensure_str -from types import NoneType - from zope.interface import implementer from twisted.application import service, strports from twisted.internet import defer @@ -317,7 +315,7 @@ class Dispatcher(object): class FTPServer(service.MultiService): def __init__(self, client, accountfile, accounturl, ftp_portstr): - precondition(isinstance(accountfile, (unicode, NoneType)), accountfile) + precondition(isinstance(accountfile, (unicode, type(None))), accountfile) service.MultiService.__init__(self) r = Dispatcher(client) From 8b991d3516166b62b7dfe4827ba7b1911de5f680 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 29 Nov 2020 15:16:45 -0500 Subject: [PATCH 032/154] Update DirectoryNode.set_uri to match interface spec. --- src/allmydata/dirnode.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/dirnode.py b/src/allmydata/dirnode.py index 59ebd73ba..6f052c3c7 100644 --- a/src/allmydata/dirnode.py +++ b/src/allmydata/dirnode.py @@ -554,7 +554,7 @@ class DirectoryNode(object): d = self.get_child_and_metadata(childnamex) return d - def set_uri(self, namex, writecap, readcap, metadata=None, overwrite=True): + def set_uri(self, namex, writecap, readcap=None, metadata=None, overwrite=True): precondition(isinstance(writecap, (str,type(None))), writecap) precondition(isinstance(readcap, (str,type(None))), readcap) From 6ea9003436e79163a865a0a30b6051f7811c1bd3 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 29 Nov 2020 15:19:11 -0500 Subject: [PATCH 033/154] Declare MultiFormatResource.formatDefault as optional string for subclass overrides. --- src/allmydata/web/common_py3.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/allmydata/web/common_py3.py b/src/allmydata/web/common_py3.py index 22f235790..080c24c19 100644 --- a/src/allmydata/web/common_py3.py +++ b/src/allmydata/web/common_py3.py @@ -4,6 +4,11 @@ Common utilities that are available from Python 3. Can eventually be merged back into allmydata.web.common. """ +try: + from typing import Optional +except ImportError: + pass + from twisted.web import resource, http from allmydata.util import abbreviate @@ -47,7 +52,7 @@ class MultiFormatResource(resource.Resource, object): format if nothing else is given as the ``formatDefault``. """ formatArgument = "t" - formatDefault = None + formatDefault = None # type: Optional[str] def render(self, req): """ From cb351607d81e21919c3c380c0ac55cd0dba3a5fc Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 29 Nov 2020 15:21:25 -0500 Subject: [PATCH 034/154] Repeat type declaration from parent to avoid over-constraining this type for subclasses. --- src/allmydata/scripts/cli.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/allmydata/scripts/cli.py b/src/allmydata/scripts/cli.py index 379e1d212..96dde65f8 100644 --- a/src/allmydata/scripts/cli.py +++ b/src/allmydata/scripts/cli.py @@ -1,6 +1,12 @@ from __future__ import print_function import os.path, re, fnmatch + +try: + from typing import List, Sequence, Any +except ImportError: + pass + from twisted.python import usage from allmydata.scripts.common import get_aliases, get_default_nodedir, \ DEFAULT_ALIAS, BaseOptions @@ -19,7 +25,7 @@ class FileStoreOptions(BaseOptions): "This overrides the URL found in the --node-directory ."], ["dir-cap", None, None, "Specify which dirnode URI should be used as the 'tahoe' alias."] - ] + ] # type: List[Sequence[Any]] def postOptions(self): self["quiet"] = self.parent["quiet"] From 6b772e7fdc14f473492a61e3e5abf507116077d0 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 29 Nov 2020 15:22:58 -0500 Subject: [PATCH 035/154] Declare type for BaseOptions.description_unwrapped. --- src/allmydata/scripts/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/scripts/common.py b/src/allmydata/scripts/common.py index 81511681d..b501c7e6a 100644 --- a/src/allmydata/scripts/common.py +++ b/src/allmydata/scripts/common.py @@ -46,7 +46,7 @@ class BaseOptions(usage.Options): raise usage.UsageError("--version not allowed on subcommands") description = None # type: Optional[str] - description_unwrapped = None + description_unwrapped = None # type: Optional[str] def __str__(self): width = int(os.environ.get('COLUMNS', '80')) From 41c341a3cc0e2a58847907cc9bee471de4d68519 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 29 Nov 2020 15:24:20 -0500 Subject: [PATCH 036/154] Prefer type(None) for better compatibility. --- src/allmydata/frontends/sftpd.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/allmydata/frontends/sftpd.py b/src/allmydata/frontends/sftpd.py index db914fa45..6ee42d505 100644 --- a/src/allmydata/frontends/sftpd.py +++ b/src/allmydata/frontends/sftpd.py @@ -1,6 +1,5 @@ import six import heapq, traceback, array, stat, struct -from types import NoneType from stat import S_IFREG, S_IFDIR from time import time, strftime, localtime @@ -267,7 +266,7 @@ def _attrs_to_metadata(attrs): def _direntry_for(filenode_or_parent, childname, filenode=None): - precondition(isinstance(childname, (unicode, NoneType)), childname=childname) + precondition(isinstance(childname, (unicode, type(None))), childname=childname) if childname is None: filenode_or_parent = filenode @@ -672,7 +671,7 @@ class GeneralSFTPFile(PrefixingLogMixin): self.log(".open(parent=%r, childname=%r, filenode=%r, metadata=%r)" % (parent, childname, filenode, metadata), level=OPERATIONAL) - precondition(isinstance(childname, (unicode, NoneType)), childname=childname) + precondition(isinstance(childname, (unicode, type(None))), childname=childname) precondition(filenode is None or IFileNode.providedBy(filenode), filenode=filenode) precondition(not self.closed, sftpfile=self) @@ -1194,7 +1193,7 @@ class SFTPUserHandler(ConchUser, PrefixingLogMixin): request = "._sync_heisenfiles(%r, %r, ignore=%r)" % (userpath, direntry, ignore) self.log(request, level=OPERATIONAL) - _assert(isinstance(userpath, str) and isinstance(direntry, (str, NoneType)), + _assert(isinstance(userpath, str) and isinstance(direntry, (str, type(None))), userpath=userpath, direntry=direntry) files = [] @@ -1219,7 +1218,7 @@ class SFTPUserHandler(ConchUser, PrefixingLogMixin): def _remove_heisenfile(self, userpath, parent, childname, file_to_remove): if noisy: self.log("._remove_heisenfile(%r, %r, %r, %r)" % (userpath, parent, childname, file_to_remove), level=NOISY) - _assert(isinstance(userpath, str) and isinstance(childname, (unicode, NoneType)), + _assert(isinstance(userpath, str) and isinstance(childname, (unicode, type(None))), userpath=userpath, childname=childname) direntry = _direntry_for(parent, childname) @@ -1246,7 +1245,7 @@ class SFTPUserHandler(ConchUser, PrefixingLogMixin): (existing_file, userpath, flags, _repr_flags(flags), parent, childname, filenode, metadata), level=NOISY) - _assert((isinstance(userpath, str) and isinstance(childname, (unicode, NoneType)) and + _assert((isinstance(userpath, str) and isinstance(childname, (unicode, type(None))) and (metadata is None or 'no-write' in metadata)), userpath=userpath, childname=childname, metadata=metadata) @@ -1977,7 +1976,7 @@ class Dispatcher(object): class SFTPServer(service.MultiService): def __init__(self, client, accountfile, accounturl, sftp_portstr, pubkey_file, privkey_file): - precondition(isinstance(accountfile, (unicode, NoneType)), accountfile) + precondition(isinstance(accountfile, (unicode, type(None))), accountfile) precondition(isinstance(pubkey_file, unicode), pubkey_file) precondition(isinstance(privkey_file, unicode), privkey_file) service.MultiService.__init__(self) From acbb6b3e93f0a1082e63db42fae20131908afe5e Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 29 Nov 2020 15:48:26 -0500 Subject: [PATCH 037/154] Convert subcommands to tuples instead of lists, as that's what mypy demands for heterogeneous sequences. --- src/allmydata/scripts/admin.py | 9 ++++-- src/allmydata/scripts/cli.py | 39 +++++++++++++------------ src/allmydata/scripts/create_node.py | 13 ++++++--- src/allmydata/scripts/debug.py | 9 ++++-- src/allmydata/scripts/runner.py | 17 +++++++---- src/allmydata/scripts/stats_gatherer.py | 9 ++++-- src/allmydata/scripts/tahoe_invite.py | 7 ++++- src/allmydata/scripts/types_.py | 10 +++++++ 8 files changed, 77 insertions(+), 36 deletions(-) create mode 100644 src/allmydata/scripts/types_.py diff --git a/src/allmydata/scripts/admin.py b/src/allmydata/scripts/admin.py index e472ffd8c..50dde9e43 100644 --- a/src/allmydata/scripts/admin.py +++ b/src/allmydata/scripts/admin.py @@ -1,5 +1,10 @@ from __future__ import print_function +try: + from allmydata.scripts.types_ import SubCommands +except ImportError: + pass + from twisted.python import usage from allmydata.scripts.common import BaseOptions @@ -79,8 +84,8 @@ def do_admin(options): subCommands = [ - ["admin", None, AdminCommand, "admin subcommands: use 'tahoe admin' for a list"], - ] + ("admin", None, AdminCommand, "admin subcommands: use 'tahoe admin' for a list"), + ] # type: SubCommands dispatch = { "admin": do_admin, diff --git a/src/allmydata/scripts/cli.py b/src/allmydata/scripts/cli.py index 96dde65f8..2bb91472f 100644 --- a/src/allmydata/scripts/cli.py +++ b/src/allmydata/scripts/cli.py @@ -4,6 +4,7 @@ import os.path, re, fnmatch try: from typing import List, Sequence, Any + from allmydata.scripts.types_ import SubCommands except ImportError: pass @@ -461,25 +462,25 @@ class DeepCheckOptions(FileStoreOptions): Optionally repair any problems found.""" subCommands = [ - ["mkdir", None, MakeDirectoryOptions, "Create a new directory."], - ["add-alias", None, AddAliasOptions, "Add a new alias cap."], - ["create-alias", None, CreateAliasOptions, "Create a new alias cap."], - ["list-aliases", None, ListAliasesOptions, "List all alias caps."], - ["ls", None, ListOptions, "List a directory."], - ["get", None, GetOptions, "Retrieve a file from the grid."], - ["put", None, PutOptions, "Upload a file into the grid."], - ["cp", None, CpOptions, "Copy one or more files or directories."], - ["unlink", None, UnlinkOptions, "Unlink a file or directory on the grid."], - ["mv", None, MvOptions, "Move a file within the grid."], - ["ln", None, LnOptions, "Make an additional link to an existing file or directory."], - ["backup", None, BackupOptions, "Make target dir look like local dir."], - ["webopen", None, WebopenOptions, "Open a web browser to a grid file or directory."], - ["manifest", None, ManifestOptions, "List all files/directories in a subtree."], - ["stats", None, StatsOptions, "Print statistics about all files/directories in a subtree."], - ["check", None, CheckOptions, "Check a single file or directory."], - ["deep-check", None, DeepCheckOptions, "Check all files/directories reachable from a starting point."], - ["status", None, TahoeStatusCommand, "Various status information."], - ] + ("mkdir", None, MakeDirectoryOptions, "Create a new directory."), + ("add-alias", None, AddAliasOptions, "Add a new alias cap."), + ("create-alias", None, CreateAliasOptions, "Create a new alias cap."), + ("list-aliases", None, ListAliasesOptions, "List all alias caps."), + ("ls", None, ListOptions, "List a directory."), + ("get", None, GetOptions, "Retrieve a file from the grid."), + ("put", None, PutOptions, "Upload a file into the grid."), + ("cp", None, CpOptions, "Copy one or more files or directories."), + ("unlink", None, UnlinkOptions, "Unlink a file or directory on the grid."), + ("mv", None, MvOptions, "Move a file within the grid."), + ("ln", None, LnOptions, "Make an additional link to an existing file or directory."), + ("backup", None, BackupOptions, "Make target dir look like local dir."), + ("webopen", None, WebopenOptions, "Open a web browser to a grid file or directory."), + ("manifest", None, ManifestOptions, "List all files/directories in a subtree."), + ("stats", None, StatsOptions, "Print statistics about all files/directories in a subtree."), + ("check", None, CheckOptions, "Check a single file or directory."), + ("deep-check", None, DeepCheckOptions, "Check all files/directories reachable from a starting point."), + ("status", None, TahoeStatusCommand, "Various status information."), + ] # type: SubCommands def mkdir(options): from allmydata.scripts import tahoe_mkdir diff --git a/src/allmydata/scripts/create_node.py b/src/allmydata/scripts/create_node.py index 2634e0915..9af9bf3ad 100644 --- a/src/allmydata/scripts/create_node.py +++ b/src/allmydata/scripts/create_node.py @@ -3,6 +3,11 @@ from __future__ import print_function import os import json +try: + from allmydata.scripts.types_ import SubCommands +except ImportError: + pass + from twisted.internet import reactor, defer from twisted.python.usage import UsageError from allmydata.scripts.common import BasedirOptions, NoDefaultBasedirOptions @@ -478,10 +483,10 @@ def create_introducer(config): subCommands = [ - ["create-node", None, CreateNodeOptions, "Create a node that acts as a client, server or both."], - ["create-client", None, CreateClientOptions, "Create a client node (with storage initially disabled)."], - ["create-introducer", None, CreateIntroducerOptions, "Create an introducer node."], -] + ("create-node", None, CreateNodeOptions, "Create a node that acts as a client, server or both."), + ("create-client", None, CreateClientOptions, "Create a client node (with storage initially disabled)."), + ("create-introducer", None, CreateIntroducerOptions, "Create an introducer node."), +] # type: SubCommands dispatch = { "create-node": create_node, diff --git a/src/allmydata/scripts/debug.py b/src/allmydata/scripts/debug.py index fd3f2b87c..be6b7e2bc 100644 --- a/src/allmydata/scripts/debug.py +++ b/src/allmydata/scripts/debug.py @@ -1,5 +1,10 @@ from __future__ import print_function +try: + from allmydata.scripts.types_ import SubCommands +except ImportError: + pass + # do not import any allmydata modules at this level. Do that from inside # individual functions instead. import struct, time, os, sys @@ -1051,8 +1056,8 @@ def do_debug(options): subCommands = [ - ["debug", None, DebugCommand, "debug subcommands: use 'tahoe debug' for a list."], - ] + ("debug", None, DebugCommand, "debug subcommands: use 'tahoe debug' for a list."), + ] # type: SubCommands dispatch = { "debug": do_debug, diff --git a/src/allmydata/scripts/runner.py b/src/allmydata/scripts/runner.py index 3436a1b84..999d7d353 100644 --- a/src/allmydata/scripts/runner.py +++ b/src/allmydata/scripts/runner.py @@ -4,6 +4,11 @@ import os, sys from six.moves import StringIO import six +try: + from allmydata.scripts.types_ import SubCommands +except ImportError: + pass + from twisted.python import usage from twisted.internet import defer, task, threads @@ -45,12 +50,12 @@ _control_node_dispatch = { } process_control_commands = [ - ["run", None, tahoe_run.RunOptions, "run a node without daemonizing"], - ["daemonize", None, tahoe_daemonize.DaemonizeOptions, "(deprecated) run a node in the background"], - ["start", None, tahoe_start.StartOptions, "(deprecated) start a node in the background and confirm it started"], - ["stop", None, tahoe_stop.StopOptions, "(deprecated) stop a node"], - ["restart", None, tahoe_restart.RestartOptions, "(deprecated) restart a node"], -] + ("run", None, tahoe_run.RunOptions, "run a node without daemonizing"), + ("daemonize", None, tahoe_daemonize.DaemonizeOptions, "(deprecated) run a node in the background"), + ("start", None, tahoe_start.StartOptions, "(deprecated) start a node in the background and confirm it started"), + ("stop", None, tahoe_stop.StopOptions, "(deprecated) stop a node"), + ("restart", None, tahoe_restart.RestartOptions, "(deprecated) restart a node"), +] # type: SubCommands class Options(usage.Options): diff --git a/src/allmydata/scripts/stats_gatherer.py b/src/allmydata/scripts/stats_gatherer.py index 26848a23c..b16ce689e 100644 --- a/src/allmydata/scripts/stats_gatherer.py +++ b/src/allmydata/scripts/stats_gatherer.py @@ -2,6 +2,11 @@ from __future__ import print_function import os +try: + from allmydata.scripts.types_ import SubCommands +except ImportError: + pass + # Python 2 compatibility from future.utils import PY2 if PY2: @@ -93,8 +98,8 @@ def create_stats_gatherer(config): return 0 subCommands = [ - ["create-stats-gatherer", None, CreateStatsGathererOptions, "Create a stats-gatherer service."], -] + ("create-stats-gatherer", None, CreateStatsGathererOptions, "Create a stats-gatherer service."), +] # type: SubCommands dispatch = { "create-stats-gatherer": create_stats_gatherer, diff --git a/src/allmydata/scripts/tahoe_invite.py b/src/allmydata/scripts/tahoe_invite.py index cca4216e3..f2d978f55 100644 --- a/src/allmydata/scripts/tahoe_invite.py +++ b/src/allmydata/scripts/tahoe_invite.py @@ -3,6 +3,11 @@ from __future__ import print_function import json from os.path import join +try: + from allmydata.scripts.types_ import SubCommands +except ImportError: + pass + from twisted.python import usage from twisted.internet import defer, reactor @@ -104,7 +109,7 @@ def invite(options): subCommands = [ ("invite", None, InviteOptions, "Invite a new node to this grid"), -] +] # type: SubCommands dispatch = { "invite": invite, diff --git a/src/allmydata/scripts/types_.py b/src/allmydata/scripts/types_.py new file mode 100644 index 000000000..58f88722b --- /dev/null +++ b/src/allmydata/scripts/types_.py @@ -0,0 +1,10 @@ +from typing import List, Tuple, Type +from allmydata.scripts.common import BaseOptions + + +# Historically, subcommands were implemented as lists, but due to a +# [designed contraint in mypy](https://stackoverflow.com/a/52559625/70170), +# a Tuple is required. +SubCommand = Tuple[str, None, Type[BaseOptions], str] + +SubCommands = List[SubCommand] From 572d7b2e02b2ba54c8c4db1592bac1898b48aba1 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 29 Nov 2020 15:52:04 -0500 Subject: [PATCH 038/154] Ignore error when untyped Module has no dispatch. --- src/allmydata/scripts/runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/scripts/runner.py b/src/allmydata/scripts/runner.py index 999d7d353..e56130d87 100644 --- a/src/allmydata/scripts/runner.py +++ b/src/allmydata/scripts/runner.py @@ -113,7 +113,7 @@ class Options(usage.Options): create_dispatch = {} for module in (create_node, stats_gatherer): - create_dispatch.update(module.dispatch) + create_dispatch.update(module.dispatch) # type: ignore def parse_options(argv, config=None): if not config: From b1b3a2341517d2b993e21f6764797aa3f6c01d4f Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 29 Nov 2020 15:56:39 -0500 Subject: [PATCH 039/154] Fix type errors with CPUUsageMonitor subclasses with float POLL_INTERVAL. --- src/allmydata/stats.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/stats.py b/src/allmydata/stats.py index f669b0861..6de323b73 100644 --- a/src/allmydata/stats.py +++ b/src/allmydata/stats.py @@ -78,7 +78,7 @@ class LoadMonitor(service.MultiService): @implementer(IStatsProducer) class CPUUsageMonitor(service.MultiService): HISTORY_LENGTH = 15 - POLL_INTERVAL = 60 + POLL_INTERVAL = 60 # type: float def __init__(self): service.MultiService.__init__(self) From 1768377aecc4d7e563e0b42d0c79d8d0f2d7d8a5 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 29 Nov 2020 16:04:56 -0500 Subject: [PATCH 040/154] Ignore error in DummyStorage --- src/allmydata/test/storage_plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/storage_plugin.py b/src/allmydata/test/storage_plugin.py index 4a1f84531..24081ae09 100644 --- a/src/allmydata/test/storage_plugin.py +++ b/src/allmydata/test/storage_plugin.py @@ -48,7 +48,7 @@ class RIDummy(RemoteInterface): -@implementer(IFoolscapStoragePlugin) +@implementer(IFoolscapStoragePlugin) # type: ignore # todo: make stubs for twisted @attr.s class DummyStorage(object): name = attr.ib() From 3653d7ed16c6b5d365664aca504af0c388e99752 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 29 Nov 2020 16:05:09 -0500 Subject: [PATCH 041/154] Ignore type checks on Referenceable objects. Ref warner/foolscap#78. --- src/allmydata/test/storage_plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/storage_plugin.py b/src/allmydata/test/storage_plugin.py index 24081ae09..b3464a88f 100644 --- a/src/allmydata/test/storage_plugin.py +++ b/src/allmydata/test/storage_plugin.py @@ -107,7 +107,7 @@ class GetCounter(Resource, object): @implementer(RIDummy) @attr.s(frozen=True) -class DummyStorageServer(object): +class DummyStorageServer(object): # type: ignore # warner/foolscap#78 get_anonymous_storage_server = attr.ib() def remote_just_some_method(self): From 2514196b275ad65b9e546b4c54898a43be949edd Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 29 Nov 2020 16:06:46 -0500 Subject: [PATCH 042/154] Suppress typing error in DummyStorageClient --- src/allmydata/test/storage_plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/storage_plugin.py b/src/allmydata/test/storage_plugin.py index b3464a88f..ba11776e6 100644 --- a/src/allmydata/test/storage_plugin.py +++ b/src/allmydata/test/storage_plugin.py @@ -116,7 +116,7 @@ class DummyStorageServer(object): # type: ignore # warner/foolscap#78 @implementer(IStorageServer) @attr.s -class DummyStorageClient(object): +class DummyStorageClient(object): # type: ignore # incomplete implementation get_rref = attr.ib() configuration = attr.ib() announcement = attr.ib() From ffa19d1c07e6a876fcebe8e8edbe968b10c842a9 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 29 Nov 2020 16:08:22 -0500 Subject: [PATCH 043/154] Suppress typing errors in common Nodes --- src/allmydata/test/common.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/common.py b/src/allmydata/test/common.py index a420dd3ba..a68a56ba5 100644 --- a/src/allmydata/test/common.py +++ b/src/allmydata/test/common.py @@ -391,7 +391,7 @@ class DummyProducer(object): pass @implementer(IImmutableFileNode) -class FakeCHKFileNode(object): +class FakeCHKFileNode(object): # type: ignore # incomplete implementation """I provide IImmutableFileNode, but all of my data is stored in a class-level dictionary.""" @@ -529,7 +529,7 @@ def create_chk_filenode(contents, all_contents): @implementer(IMutableFileNode, ICheckable) -class FakeMutableFileNode(object): +class FakeMutableFileNode(object): # type: ignore # incomplete implementation """I provide IMutableFileNode, but all of my data is stored in a class-level dictionary.""" From 53ff740f0ec2a13b50673fb989190cc73bee60f6 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 29 Nov 2020 16:14:10 -0500 Subject: [PATCH 044/154] Suppress type check error on NativeStorageServerWithVersion --- src/allmydata/test/test_storage_client.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/allmydata/test/test_storage_client.py b/src/allmydata/test/test_storage_client.py index fa3a34b15..f2be9ad1e 100644 --- a/src/allmydata/test/test_storage_client.py +++ b/src/allmydata/test/test_storage_client.py @@ -90,7 +90,12 @@ from allmydata.interfaces import ( SOME_FURL = b"pb://abcde@nowhere/fake" -class NativeStorageServerWithVersion(NativeStorageServer): + +# type checks fail with: +# Cannot determine consistent method resolution order (MRO) for "NativeStorageServerWithVersion" +# even though class hierarchy is single-inheritance. Probably `implementer` +# wrappers are affecting the MRO. +class NativeStorageServerWithVersion(NativeStorageServer): # type: ignore def __init__(self, version): # note: these instances won't work for anything other than # get_available_space() because we don't upcall From 7507e84a18902f5874a9aaaabf45dee92ec05510 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 29 Nov 2020 16:19:24 -0500 Subject: [PATCH 045/154] Suppress errors in no_network --- src/allmydata/test/no_network.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/allmydata/test/no_network.py b/src/allmydata/test/no_network.py index 59ab807bb..1e8c519e1 100644 --- a/src/allmydata/test/no_network.py +++ b/src/allmydata/test/no_network.py @@ -67,7 +67,7 @@ class Marker(object): fireNow = partial(defer.succeed, None) -@implementer(IRemoteReference) +@implementer(IRemoteReference) # type: ignore # todo: write stubs for foolscap class LocalWrapper(object): """ A ``LocalWrapper`` presents the remote reference interface to a local @@ -212,9 +212,12 @@ class NoNetworkServer(object): return _StorageServer(lambda: self.rref) def get_version(self): return self.rref.version + def start_connecting(self, trigger_cb): + raise NotImplementedError + @implementer(IStorageBroker) -class NoNetworkStorageBroker(object): +class NoNetworkStorageBroker(object): # type: ignore # missing many methods def get_servers_for_psi(self, peer_selection_index): def _permuted(server): seed = server.get_permutation_seed() @@ -258,7 +261,7 @@ def create_no_network_client(basedir): return defer.succeed(client) -class _NoNetworkClient(_Client): +class _NoNetworkClient(_Client): # type: ignore # Cannot determine consistent MRO order """ Overrides all _Client networking functionality to do nothing. """ From 13cd780231bf3185104d930d748c75b9d324eabc Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 29 Nov 2020 16:22:12 -0500 Subject: [PATCH 046/154] Prefer sys.maxsize to sys.maxint. --- src/allmydata/test/check_memory.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/allmydata/test/check_memory.py b/src/allmydata/test/check_memory.py index 41cf6e1d7..4f50b383b 100644 --- a/src/allmydata/test/check_memory.py +++ b/src/allmydata/test/check_memory.py @@ -499,13 +499,13 @@ if __name__ == '__main__': mode = "upload" if len(sys.argv) > 1: mode = sys.argv[1] - if sys.maxint == 2147483647: + if sys.maxsize == 2147483647: bits = "32" - elif sys.maxint == 9223372036854775807: + elif sys.maxsize == 9223372036854775807: bits = "64" else: bits = "?" - print("%s-bit system (sys.maxint=%d)" % (bits, sys.maxint)) + print("%s-bit system (sys.maxsize=%d)" % (bits, sys.maxsize)) # put the logfile and stats.out in _test_memory/ . These stick around. # put the nodes and other files in _test_memory/test/ . These are # removed each time we run. From b0803a2ac054234f341ec1650695ce988d926f25 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 29 Nov 2020 16:24:20 -0500 Subject: [PATCH 047/154] Suppress errors in test_web due to ambiguous MRO --- src/allmydata/test/web/test_web.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/web/test_web.py b/src/allmydata/test/web/test_web.py index 326569a26..a5fbe5a51 100644 --- a/src/allmydata/test/web/test_web.py +++ b/src/allmydata/test/web/test_web.py @@ -189,7 +189,7 @@ class FakeHistory(object): def list_all_helper_statuses(self): return [] -class FakeDisplayableServer(StubServer): +class FakeDisplayableServer(StubServer): # type: ignore # Cannot determine MRO def __init__(self, serverid, nickname, connected, last_connect_time, last_loss_time, last_rx_time): StubServer.__init__(self, serverid) @@ -255,7 +255,7 @@ class FakeStorageServer(service.MultiService): def on_status_changed(self, cb): cb(self) -class FakeClient(_Client): +class FakeClient(_Client): # type: ignore # Cannot determine MRO def __init__(self): # don't upcall to Client.__init__, since we only want to initialize a # minimal subset From fc19d1baf4a503dabc1a69b93adfd3200b5e413b Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 29 Nov 2020 16:26:36 -0500 Subject: [PATCH 048/154] Suppress errors in test_sftp --- src/allmydata/test/test_sftp.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/allmydata/test/test_sftp.py b/src/allmydata/test/test_sftp.py index b6f1fbc8a..1ff0363e8 100644 --- a/src/allmydata/test/test_sftp.py +++ b/src/allmydata/test/test_sftp.py @@ -9,18 +9,15 @@ from twisted.python.failure import Failure from twisted.internet.error import ProcessDone, ProcessTerminated from allmydata.util import deferredutil -conch_interfaces = None -sftp = None -sftpd = None - try: from twisted.conch import interfaces as conch_interfaces from twisted.conch.ssh import filetransfer as sftp from allmydata.frontends import sftpd except ImportError as e: + conch_interfaces = sftp = sftpd = None # type: ignore conch_unavailable_reason = e else: - conch_unavailable_reason = None + conch_unavailable_reason = None # type: ignore from allmydata.interfaces import IDirectoryNode, ExistingChildError, NoSuchChildError from allmydata.mutable.common import NotWriteableError From 54e45498367485d44ed6f886c50feb269a26c9e5 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 29 Nov 2020 16:30:13 -0500 Subject: [PATCH 049/154] Satisfy type check in test_helper.FakeClient. --- src/allmydata/test/test_helper.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/allmydata/test/test_helper.py b/src/allmydata/test/test_helper.py index 65c07135a..3faffbe0d 100644 --- a/src/allmydata/test/test_helper.py +++ b/src/allmydata/test/test_helper.py @@ -19,6 +19,12 @@ from functools import ( ) import attr +try: + from typing import List + from allmydata.introducer.client import IntroducerClient +except ImportError: + pass + from twisted.internet import defer from twisted.trial import unittest from twisted.application import service @@ -125,7 +131,7 @@ class FakeCHKCheckerAndUEBFetcher(object): )) class FakeClient(service.MultiService): - introducer_clients = [] + introducer_clients = [] # type: List[IntroducerClient] DEFAULT_ENCODING_PARAMETERS = {"k":25, "happy": 75, "n": 100, From 86f88a4aa5568876989f1bc222703cc9dd3d35fb Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 29 Nov 2020 16:32:19 -0500 Subject: [PATCH 050/154] Satisfy type checks in test_dirnode --- src/allmydata/test/test_dirnode.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/test_dirnode.py b/src/allmydata/test/test_dirnode.py index 48ffff45a..8af1567c9 100644 --- a/src/allmydata/test/test_dirnode.py +++ b/src/allmydata/test/test_dirnode.py @@ -1526,7 +1526,7 @@ class Packing(testutil.ReallyEqualMixin, unittest.TestCase): kids, fn.get_writekey(), deep_immutable=True) @implementer(IMutableFileNode) -class FakeMutableFile(object): +class FakeMutableFile(object): # type: ignore # incomplete interface counter = 0 def __init__(self, initial_contents=""): data = self._get_initial_contents(initial_contents) @@ -1587,7 +1587,7 @@ class FakeNodeMaker(NodeMaker): def create_mutable_file(self, contents="", keysize=None, version=None): return defer.succeed(FakeMutableFile(contents)) -class FakeClient2(_Client): +class FakeClient2(_Client): # type: ignore # ambiguous MRO def __init__(self): self.nodemaker = FakeNodeMaker(None, None, None, None, None, From d2d3f1f4a9a6aecb482859c96e5732c41847051f Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 29 Nov 2020 16:33:25 -0500 Subject: [PATCH 051/154] Suppress type errors in test_checker --- src/allmydata/test/test_checker.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/allmydata/test/test_checker.py b/src/allmydata/test/test_checker.py index 85b894b1f..23ce44762 100644 --- a/src/allmydata/test/test_checker.py +++ b/src/allmydata/test/test_checker.py @@ -62,7 +62,7 @@ class FakeClient(object): @implementer(IServer) -class FakeServer(object): +class FakeServer(object): # type: ignore # incomplete interface def get_name(self): return "fake name" @@ -75,7 +75,7 @@ class FakeServer(object): @implementer(ICheckResults) -class FakeCheckResults(object): +class FakeCheckResults(object): # type: ignore # incomplete interface def __init__(self, si=None, healthy=False, recoverable=False, @@ -106,7 +106,7 @@ class FakeCheckResults(object): @implementer(ICheckAndRepairResults) -class FakeCheckAndRepairResults(object): +class FakeCheckAndRepairResults(object): # type: ignore # incomplete interface def __init__(self, si=None, repair_attempted=False, From d2e2a22f6230f880c133f05cad73bd9b8490a909 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 4 Dec 2020 10:34:21 -0500 Subject: [PATCH 052/154] Run typechecks in CircleCI --- .circleci/config.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index afa3fafa1..ff14d6dd3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -91,6 +91,9 @@ workflows: - "build-porting-depgraph": <<: *DOCKERHUB_CONTEXT + - "typechecks": + <<: *DOCKERHUB_CONTEXT + images: # Build the Docker images used by the ci jobs. This makes the ci jobs # faster and takes various spurious failures out of the critical path. @@ -475,6 +478,17 @@ jobs: . /tmp/venv/bin/activate ./misc/python3/depgraph.sh + typechecks: + docker: + - <<: *DOCKERHUB_AUTH + image: "jaraco/multipy-tox" + + steps: + - "checkout" + - run: + name: "Validate Types" + command: tox -e typechecks + build-image: &BUILD_IMAGE # This is a template for a job to build a Docker image that has as much of # the setup as we can manage already done and baked in. This cuts down on From 3eb975748a9a6c32c7972ce4c498ec5afabfda9d Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 4 Dec 2020 19:56:51 -0500 Subject: [PATCH 053/154] Ignore type checks in allmydata. --- src/allmydata/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/__init__.py b/src/allmydata/__init__.py index 15d5fb240..a43781158 100644 --- a/src/allmydata/__init__.py +++ b/src/allmydata/__init__.py @@ -14,7 +14,7 @@ __all__ = [ __version__ = "unknown" try: - from allmydata._version import __version__ + from allmydata._version import __version__ # type: ignore except ImportError: # We're running in a tree that hasn't run update_version, and didn't # come with a _version.py, so we don't know what our version is. @@ -24,7 +24,7 @@ except ImportError: full_version = "unknown" branch = "unknown" try: - from allmydata._version import full_version, branch + from allmydata._version import full_version, branch # type: ignore except ImportError: # We're running in a tree that hasn't run update_version, and didn't # come with a _version.py, so we don't know what our full version or From 8d439fd58d02366dff9b38204eeb67c204d39861 Mon Sep 17 00:00:00 2001 From: viktoriiasavchuk <75451912+viktoriiasavchuk@users.noreply.github.com> Date: Sun, 13 Dec 2020 20:59:58 +0200 Subject: [PATCH 054/154] Update CONTRIBUTING.rst Added link to the Contributor Code of Conduct, https://github.com/tahoe-lafs/tahoe-lafs/blob/master/docs/CODE_OF_CONDUCT.md --- .github/CONTRIBUTING.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/CONTRIBUTING.rst b/.github/CONTRIBUTING.rst index c8f5093f1..ad1f1b5f3 100644 --- a/.github/CONTRIBUTING.rst +++ b/.github/CONTRIBUTING.rst @@ -17,4 +17,4 @@ Examples of contributions include: * `Patch reviews `_ Before authoring or reviewing a patch, -please familiarize yourself with the `coding standard `_. +please familiarize yourself with the `Coding Standard `_ and the `Contributor Code of Conduct `_. From 3a0b72f34c262fa9a6493690be13bbe8b4648b7d Mon Sep 17 00:00:00 2001 From: Viktoriia <75451912+viktoriiasavchuk@users.noreply.github.com> Date: Tue, 15 Dec 2020 14:11:26 +0200 Subject: [PATCH 055/154] Update CONTRIBUTING.rst Changed 'Coding Standard' to 'Coding Standards' --- .github/CONTRIBUTING.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/CONTRIBUTING.rst b/.github/CONTRIBUTING.rst index ad1f1b5f3..0c0da9503 100644 --- a/.github/CONTRIBUTING.rst +++ b/.github/CONTRIBUTING.rst @@ -17,4 +17,4 @@ Examples of contributions include: * `Patch reviews `_ Before authoring or reviewing a patch, -please familiarize yourself with the `Coding Standard `_ and the `Contributor Code of Conduct `_. +please familiarize yourself with the `Coding Standards `_ and the `Contributor Code of Conduct `_. From 950ca189326703daf8bb84188507ca7324efc16e Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 18 Dec 2020 09:23:27 -0500 Subject: [PATCH 056/154] Ignores no longer needed. --- src/allmydata/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/__init__.py b/src/allmydata/__init__.py index a43781158..15d5fb240 100644 --- a/src/allmydata/__init__.py +++ b/src/allmydata/__init__.py @@ -14,7 +14,7 @@ __all__ = [ __version__ = "unknown" try: - from allmydata._version import __version__ # type: ignore + from allmydata._version import __version__ except ImportError: # We're running in a tree that hasn't run update_version, and didn't # come with a _version.py, so we don't know what our version is. @@ -24,7 +24,7 @@ except ImportError: full_version = "unknown" branch = "unknown" try: - from allmydata._version import full_version, branch # type: ignore + from allmydata._version import full_version, branch except ImportError: # We're running in a tree that hasn't run update_version, and didn't # come with a _version.py, so we don't know what our full version or From 99da74fffb1b013602ae847dfd4756ccb5e61c3d Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 18 Dec 2020 09:58:56 -0500 Subject: [PATCH 057/154] Change comment to clarify that it's the implementation that's incomplete. --- src/allmydata/test/test_checker.py | 6 +++--- src/allmydata/test/test_dirnode.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/allmydata/test/test_checker.py b/src/allmydata/test/test_checker.py index 23ce44762..d24ad4ef4 100644 --- a/src/allmydata/test/test_checker.py +++ b/src/allmydata/test/test_checker.py @@ -62,7 +62,7 @@ class FakeClient(object): @implementer(IServer) -class FakeServer(object): # type: ignore # incomplete interface +class FakeServer(object): # type: ignore # incomplete implementation def get_name(self): return "fake name" @@ -75,7 +75,7 @@ class FakeServer(object): # type: ignore # incomplete interface @implementer(ICheckResults) -class FakeCheckResults(object): # type: ignore # incomplete interface +class FakeCheckResults(object): # type: ignore # incomplete implementation def __init__(self, si=None, healthy=False, recoverable=False, @@ -106,7 +106,7 @@ class FakeCheckResults(object): # type: ignore # incomplete interface @implementer(ICheckAndRepairResults) -class FakeCheckAndRepairResults(object): # type: ignore # incomplete interface +class FakeCheckAndRepairResults(object): # type: ignore # incomplete implementation def __init__(self, si=None, repair_attempted=False, diff --git a/src/allmydata/test/test_dirnode.py b/src/allmydata/test/test_dirnode.py index 8af1567c9..68fcdd54b 100644 --- a/src/allmydata/test/test_dirnode.py +++ b/src/allmydata/test/test_dirnode.py @@ -1526,7 +1526,7 @@ class Packing(testutil.ReallyEqualMixin, unittest.TestCase): kids, fn.get_writekey(), deep_immutable=True) @implementer(IMutableFileNode) -class FakeMutableFile(object): # type: ignore # incomplete interface +class FakeMutableFile(object): # type: ignore # incomplete implementation counter = 0 def __init__(self, initial_contents=""): data = self._get_initial_contents(initial_contents) From 51b0b201b49733280702ed901baa06e23d75af85 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 18 Dec 2020 10:33:30 -0500 Subject: [PATCH 058/154] Expand comment to provide more context. --- src/allmydata/web/private.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/allmydata/web/private.py b/src/allmydata/web/private.py index a86c869c4..405ca75e7 100644 --- a/src/allmydata/web/private.py +++ b/src/allmydata/web/private.py @@ -61,7 +61,12 @@ class IToken(ICredentials): pass -# Shoobx/mypy-zope#26 +# Workaround for Shoobx/mypy-zope#26, where without suitable +# stubs for twisted classes (ICredentials), IToken does not +# appear to be an Interface. The proper fix appears to be to +# create stubs for twisted +# (https://twistedmatrix.com/trac/ticket/9717). For now, +# bypassing the inline decorator syntax works around the issue. _itoken_impl = implementer(IToken) From efd0aef2847814c1c43a14db3ccc1cde01a95119 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 18 Dec 2020 10:36:43 -0500 Subject: [PATCH 059/154] Indicate that unichr is Python 2 only. --- src/allmydata/windows/fixups.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/windows/fixups.py b/src/allmydata/windows/fixups.py index c5ba3bb57..e98aa8a67 100644 --- a/src/allmydata/windows/fixups.py +++ b/src/allmydata/windows/fixups.py @@ -219,7 +219,7 @@ def initialize(): def unmangle(s): return re.sub( u'\\x7F[0-9a-fA-F]*\\;', - # type ignored for 'unichr' + # type ignored for 'unichr' (Python 2 only) lambda m: unichr(int(m.group(0)[1:-1], 16)), # type: ignore s, ) From ea0c10ef83d98f019472c744ad81f3c5a8f5fab1 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 18 Dec 2020 10:51:56 -0500 Subject: [PATCH 060/154] Remove set_size, unused --- src/allmydata/immutable/encode.py | 3 --- src/allmydata/interfaces.py | 5 ----- 2 files changed, 8 deletions(-) diff --git a/src/allmydata/immutable/encode.py b/src/allmydata/immutable/encode.py index e743ce766..9351df501 100644 --- a/src/allmydata/immutable/encode.py +++ b/src/allmydata/immutable/encode.py @@ -711,6 +711,3 @@ class Encoder(object): return self.uri_extension_data def get_uri_extension_hash(self): return self.uri_extension_hash - - def set_size(self, size): - raise NotImplementedError() diff --git a/src/allmydata/interfaces.py b/src/allmydata/interfaces.py index 95b1fdf63..edd16ea08 100644 --- a/src/allmydata/interfaces.py +++ b/src/allmydata/interfaces.py @@ -1825,11 +1825,6 @@ class IEncoder(Interface): willing to receive data. """ - def set_size(size): - """Specify the number of bytes that will be encoded. This must be - peformed before get_serialized_params() can be called. - """ - def set_encrypted_uploadable(u): """Provide a source of encrypted upload data. 'u' must implement IEncryptedUploadable. From d051791e9530e5e64d49942cdfd6afc0a9a98c17 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 18 Dec 2020 10:53:24 -0500 Subject: [PATCH 061/154] Add reference to ticket. --- src/allmydata/immutable/offloaded.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/allmydata/immutable/offloaded.py b/src/allmydata/immutable/offloaded.py index 53a1f911a..2d2c5c1f5 100644 --- a/src/allmydata/immutable/offloaded.py +++ b/src/allmydata/immutable/offloaded.py @@ -499,6 +499,7 @@ class LocalCiphertextReader(AskUntilSuccessMixin): # ??. I'm not sure if it makes sense to forward the close message. return self.call("close") + # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3561 def set_upload_status(self, upload_status): raise NotImplementedError From 090031cbfc33510e7eebc627f313c94c548ea91e Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 18 Dec 2020 10:55:20 -0500 Subject: [PATCH 062/154] Remove confirm_share_allocation from interface (unused). --- src/allmydata/immutable/upload.py | 3 --- src/allmydata/interfaces.py | 6 ------ 2 files changed, 9 deletions(-) diff --git a/src/allmydata/immutable/upload.py b/src/allmydata/immutable/upload.py index 171b71eff..e6da4812a 100644 --- a/src/allmydata/immutable/upload.py +++ b/src/allmydata/immutable/upload.py @@ -388,9 +388,6 @@ class PeerSelector(object): def add_peers(self, peerids=None): raise NotImplementedError - def confirm_share_allocation(self, peerid, shnum): - raise NotImplementedError - class _QueryStatistics(object): diff --git a/src/allmydata/interfaces.py b/src/allmydata/interfaces.py index edd16ea08..e460854f8 100644 --- a/src/allmydata/interfaces.py +++ b/src/allmydata/interfaces.py @@ -860,12 +860,6 @@ class IPeerSelector(Interface): peer selection begins. """ - def confirm_share_allocation(peerid, shnum): - """ - Confirm that an allocated peer=>share pairing has been - successfully established. - """ - def add_peers(peerids=set): """ Update my internal state to include the peers in peerids as From 0e248cb4ef03f40da7e29c1bcd8a1c390ea00105 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 18 Dec 2020 10:59:53 -0500 Subject: [PATCH 063/154] Declare signing key as required in introducer client publish. --- src/allmydata/introducer/client.py | 2 +- src/allmydata/introducer/interfaces.py | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/allmydata/introducer/client.py b/src/allmydata/introducer/client.py index 62642d0af..0e0a56442 100644 --- a/src/allmydata/introducer/client.py +++ b/src/allmydata/introducer/client.py @@ -198,7 +198,7 @@ class IntroducerClient(service.Service, Referenceable): ann_d.update(ann) return ann_d - def publish(self, service_name, ann, signing_key=None): + def publish(self, service_name, ann, signing_key): # we increment the seqnum every time we publish something new current_seqnum, current_nonce = self._sequencer() diff --git a/src/allmydata/introducer/interfaces.py b/src/allmydata/introducer/interfaces.py index 9f08f1943..24fd3945f 100644 --- a/src/allmydata/introducer/interfaces.py +++ b/src/allmydata/introducer/interfaces.py @@ -73,7 +73,7 @@ class IIntroducerClient(Interface): publish their services to the rest of the world, and I help them learn about services available on other nodes.""" - def publish(service_name, ann, signing_key=None): + def publish(service_name, ann, signing_key): """Publish the given announcement dictionary (which must be JSON-serializable), plus some additional keys, to the world. @@ -83,8 +83,7 @@ class IIntroducerClient(Interface): the signing_key, if present, otherwise it is derived from the 'anonymous-storage-FURL' key. - If signing_key= is set to an instance of SigningKey, it will be - used to sign the announcement.""" + signing_key (a SigningKey) will be used to sign the announcement.""" def subscribe_to(service_name, callback, *args, **kwargs): """Call this if you will eventually want to use services with the From c2d2aba83f025824e23f2051700ad2fd54063799 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 18 Dec 2020 11:05:55 -0500 Subject: [PATCH 064/154] Add reference to ticket. --- src/allmydata/mutable/filenode.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/allmydata/mutable/filenode.py b/src/allmydata/mutable/filenode.py index 4613a918b..39e8b76be 100644 --- a/src/allmydata/mutable/filenode.py +++ b/src/allmydata/mutable/filenode.py @@ -1206,5 +1206,6 @@ class MutableFileVersion(object): mode=mode) return u.update() + # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3562 def get_servermap(self): raise NotImplementedError From 189608e11388988b512aa00f225911ff61afc80c Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 18 Dec 2020 11:07:30 -0500 Subject: [PATCH 065/154] Remove GENERATED_FILES, unused --- src/allmydata/node.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/allmydata/node.py b/src/allmydata/node.py index 7622d5bc3..6d0b2da04 100644 --- a/src/allmydata/node.py +++ b/src/allmydata/node.py @@ -713,7 +713,6 @@ class Node(service.MultiService): """ NODETYPE = "unknown NODETYPE" CERTFILE = "node.pem" - GENERATED_FILES = [] # type: ignore def __init__(self, config, main_tub, control_tub, i2p_provider, tor_provider): """ From 602a06e5cba818cb927c45566a610eb3e0d6eadc Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 18 Dec 2020 11:14:07 -0500 Subject: [PATCH 066/154] Extract Parameters type in scripts.types_. --- src/allmydata/scripts/cli.py | 5 ++--- src/allmydata/scripts/common.py | 7 ++++--- src/allmydata/scripts/types_.py | 4 +++- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/src/allmydata/scripts/cli.py b/src/allmydata/scripts/cli.py index 2bb91472f..e4cd8aa22 100644 --- a/src/allmydata/scripts/cli.py +++ b/src/allmydata/scripts/cli.py @@ -3,8 +3,7 @@ from __future__ import print_function import os.path, re, fnmatch try: - from typing import List, Sequence, Any - from allmydata.scripts.types_ import SubCommands + from allmydata.scripts.types_ import SubCommands, Parameters except ImportError: pass @@ -26,7 +25,7 @@ class FileStoreOptions(BaseOptions): "This overrides the URL found in the --node-directory ."], ["dir-cap", None, None, "Specify which dirnode URI should be used as the 'tahoe' alias."] - ] # type: List[Sequence[Any]] + ] # type: Parameters def postOptions(self): self["quiet"] = self.parent["quiet"] diff --git a/src/allmydata/scripts/common.py b/src/allmydata/scripts/common.py index b501c7e6a..42c26bb90 100644 --- a/src/allmydata/scripts/common.py +++ b/src/allmydata/scripts/common.py @@ -5,7 +5,8 @@ import codecs from os.path import join try: - from typing import Optional, Sequence, List, Any + from typing import Optional + from .types_ import Parameters except ImportError: pass @@ -69,7 +70,7 @@ class BasedirOptions(BaseOptions): optParameters = [ ["basedir", "C", None, "Specify which Tahoe base directory should be used. [default: %s]" % quote_local_unicode_path(_default_nodedir)], - ] # type: List[Sequence[Any]] + ] # type: Parameters def parseArgs(self, basedir=None): # This finds the node-directory option correctly even if we are in a subcommand. @@ -106,7 +107,7 @@ class NoDefaultBasedirOptions(BasedirOptions): optParameters = [ ["basedir", "C", None, "Specify which Tahoe base directory should be used."], - ] # type: List[Sequence[Any]] + ] # type: Parameters # This is overridden in order to ensure we get a "Wrong number of arguments." # error when more than one argument is given. diff --git a/src/allmydata/scripts/types_.py b/src/allmydata/scripts/types_.py index 58f88722b..3937cb803 100644 --- a/src/allmydata/scripts/types_.py +++ b/src/allmydata/scripts/types_.py @@ -1,4 +1,4 @@ -from typing import List, Tuple, Type +from typing import List, Tuple, Type, Sequence, Any from allmydata.scripts.common import BaseOptions @@ -8,3 +8,5 @@ from allmydata.scripts.common import BaseOptions SubCommand = Tuple[str, None, Type[BaseOptions], str] SubCommands = List[SubCommand] + +Parameters = List[Sequence[Any]] From ff182e69c1bc699503f38477c75197671f0e67ae Mon Sep 17 00:00:00 2001 From: meejah Date: Sun, 20 Dec 2020 19:10:00 -0700 Subject: [PATCH 067/154] signatures are detached --- docs/release-checklist.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/release-checklist.rst b/docs/release-checklist.rst index be32aea6c..18c908a99 100644 --- a/docs/release-checklist.rst +++ b/docs/release-checklist.rst @@ -97,10 +97,10 @@ they will need to evaluate which contributors' signatures they trust. - install each in a fresh virtualenv - run `tahoe` command - when satisfied, sign the tarballs: - - gpg --pinentry=loopback --armor --sign dist/tahoe_lafs-1.15.0rc0-py2-none-any.whl - - gpg --pinentry=loopback --armor --sign dist/tahoe_lafs-1.15.0rc0.tar.bz2 - - gpg --pinentry=loopback --armor --sign dist/tahoe_lafs-1.15.0rc0.tar.gz - - gpg --pinentry=loopback --armor --sign dist/tahoe_lafs-1.15.0rc0.zip + - gpg --pinentry=loopback --armor --detach-sign dist/tahoe_lafs-1.15.0rc0-py2-none-any.whl + - gpg --pinentry=loopback --armor --detach-sign dist/tahoe_lafs-1.15.0rc0.tar.bz2 + - gpg --pinentry=loopback --armor --detach-sign dist/tahoe_lafs-1.15.0rc0.tar.gz + - gpg --pinentry=loopback --armor --detach-sign dist/tahoe_lafs-1.15.0rc0.zip Privileged Contributor From 6b6b8f8378f1e3da826a50d9dcbe95e63ec8faa1 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 26 Dec 2020 11:17:35 -0500 Subject: [PATCH 068/154] Push IURI implementers down to the classes that actually implement it. --- src/allmydata/uri.py | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/src/allmydata/uri.py b/src/allmydata/uri.py index d7f4782cd..ce7794dda 100644 --- a/src/allmydata/uri.py +++ b/src/allmydata/uri.py @@ -494,7 +494,6 @@ class MDMFVerifierURI(_BaseURI): return self -@implementer(IURI, IDirnodeURI) class _DirectoryBaseURI(_BaseURI): def __init__(self, filenode_uri=None): self._filenode_uri = filenode_uri @@ -540,14 +539,8 @@ class _DirectoryBaseURI(_BaseURI): def get_storage_index(self): return self._filenode_uri.get_storage_index() - def get_readonly(self): - raise NotImplementedError() - def is_readonly(self): - raise NotImplementedError() - - -@implementer(IDirectoryURI) +@implementer(IURI, IDirectoryURI) class DirectoryURI(_DirectoryBaseURI): BASE_STRING=b'URI:DIR2:' @@ -566,7 +559,7 @@ class DirectoryURI(_DirectoryBaseURI): return ReadonlyDirectoryURI(self._filenode_uri.get_readonly()) -@implementer(IReadonlyDirectoryURI) +@implementer(IURI, IReadonlyDirectoryURI) class ReadonlyDirectoryURI(_DirectoryBaseURI): BASE_STRING=b'URI:DIR2-RO:' @@ -585,6 +578,7 @@ class ReadonlyDirectoryURI(_DirectoryBaseURI): return self +@implementer(IURI, IDirnodeURI) class _ImmutableDirectoryBaseURI(_DirectoryBaseURI): def __init__(self, filenode_uri=None): if filenode_uri: @@ -622,7 +616,7 @@ class LiteralDirectoryURI(_ImmutableDirectoryBaseURI): return None -@implementer(IDirectoryURI) +@implementer(IURI, IDirectoryURI) class MDMFDirectoryURI(_DirectoryBaseURI): BASE_STRING=b'URI:DIR2-MDMF:' @@ -644,7 +638,7 @@ class MDMFDirectoryURI(_DirectoryBaseURI): return MDMFDirectoryURIVerifier(self._filenode_uri.get_verify_cap()) -@implementer(IReadonlyDirectoryURI) +@implementer(IURI, IReadonlyDirectoryURI) class ReadonlyMDMFDirectoryURI(_DirectoryBaseURI): BASE_STRING=b'URI:DIR2-MDMF-RO:' @@ -682,7 +676,7 @@ def wrap_dirnode_cap(filecap): raise AssertionError("cannot interpret as a directory cap: %s" % filecap.__class__) -@implementer(IVerifierURI) +@implementer(IURI, IVerifierURI) class MDMFDirectoryURIVerifier(_DirectoryBaseURI): BASE_STRING=b'URI:DIR2-MDMF-Verifier:' @@ -707,7 +701,7 @@ class MDMFDirectoryURIVerifier(_DirectoryBaseURI): return self -@implementer(IVerifierURI) +@implementer(IURI, IVerifierURI) class DirectoryURIVerifier(_DirectoryBaseURI): BASE_STRING=b'URI:DIR2-Verifier:' From b65ef3cee6c6922c70edac842e28985ac7c20990 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 26 Dec 2020 11:22:39 -0500 Subject: [PATCH 069/154] Revert "Ignores no longer needed." This reverts commit 950ca189326703daf8bb84188507ca7324efc16e. --- src/allmydata/__init__.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/allmydata/__init__.py b/src/allmydata/__init__.py index 15d5fb240..3157c8c80 100644 --- a/src/allmydata/__init__.py +++ b/src/allmydata/__init__.py @@ -14,7 +14,9 @@ __all__ = [ __version__ = "unknown" try: - from allmydata._version import __version__ + # type ignored as it fails in CI + # (https://app.circleci.com/pipelines/github/tahoe-lafs/tahoe-lafs/1647/workflows/60ae95d4-abe8-492c-8a03-1ad3b9e42ed3/jobs/40972) + from allmydata._version import __version__ # type: ignore except ImportError: # We're running in a tree that hasn't run update_version, and didn't # come with a _version.py, so we don't know what our version is. @@ -24,7 +26,9 @@ except ImportError: full_version = "unknown" branch = "unknown" try: - from allmydata._version import full_version, branch + # type ignored as it fails in CI + # (https://app.circleci.com/pipelines/github/tahoe-lafs/tahoe-lafs/1647/workflows/60ae95d4-abe8-492c-8a03-1ad3b9e42ed3/jobs/40972) + from allmydata._version import full_version, branch # type: ignore except ImportError: # We're running in a tree that hasn't run update_version, and didn't # come with a _version.py, so we don't know what our full version or From ab2c544efcaad629f2d49173bfdc82e8de94d944 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 26 Dec 2020 11:50:06 -0500 Subject: [PATCH 070/154] Restore IDirnodeURI --- src/allmydata/uri.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/allmydata/uri.py b/src/allmydata/uri.py index ce7794dda..51671b0ac 100644 --- a/src/allmydata/uri.py +++ b/src/allmydata/uri.py @@ -494,6 +494,7 @@ class MDMFVerifierURI(_BaseURI): return self +@implementer(IDirnodeURI) class _DirectoryBaseURI(_BaseURI): def __init__(self, filenode_uri=None): self._filenode_uri = filenode_uri From dacdf7f12da89e214eec9b0543b15ef450b27ae3 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 26 Dec 2020 11:56:13 -0500 Subject: [PATCH 071/154] Add more detail and link to upstream issue for Twisted stubs. --- src/allmydata/test/storage_plugin.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/storage_plugin.py b/src/allmydata/test/storage_plugin.py index ba11776e6..17ec89078 100644 --- a/src/allmydata/test/storage_plugin.py +++ b/src/allmydata/test/storage_plugin.py @@ -47,8 +47,9 @@ class RIDummy(RemoteInterface): """ - -@implementer(IFoolscapStoragePlugin) # type: ignore # todo: make stubs for twisted +# type ignored due to missing stubs for Twisted +# https://twistedmatrix.com/trac/ticket/9717 +@implementer(IFoolscapStoragePlugin) # type: ignore @attr.s class DummyStorage(object): name = attr.ib() From 5396f9f97eaf5a541c2334eaf8ebdb5a2d3484f5 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 26 Dec 2020 12:02:47 -0500 Subject: [PATCH 072/154] Replace fixme with reference to foolscap issue. --- src/allmydata/storage/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py index b7df702d5..5f2ef3ac2 100644 --- a/src/allmydata/storage/server.py +++ b/src/allmydata/storage/server.py @@ -581,7 +581,7 @@ class StorageServer(service.MultiService, Referenceable): for share in six.viewvalues(shares): share.add_or_renew_lease(lease_info) - def slot_testv_and_readv_and_writev( # type: ignore # fixme + def slot_testv_and_readv_and_writev( # type: ignore # warner/foolscap#78 self, storage_index, secrets, From 1bf71fd69010a0a075f66076156f0e951042b8dd Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 26 Dec 2020 12:12:39 -0500 Subject: [PATCH 073/154] Replace todo with a ticket. --- src/allmydata/test/no_network.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/no_network.py b/src/allmydata/test/no_network.py index 1e8c519e1..ba497f81a 100644 --- a/src/allmydata/test/no_network.py +++ b/src/allmydata/test/no_network.py @@ -67,7 +67,7 @@ class Marker(object): fireNow = partial(defer.succeed, None) -@implementer(IRemoteReference) # type: ignore # todo: write stubs for foolscap +@implementer(IRemoteReference) # type: ignore # warner/foolscap#79 class LocalWrapper(object): """ A ``LocalWrapper`` presents the remote reference interface to a local From 01147f4627a10ecb4db5ddcdcb4a17f5d839a627 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 26 Dec 2020 12:33:22 -0500 Subject: [PATCH 074/154] Add reference to ticket for ambiguous MRO --- src/allmydata/test/no_network.py | 2 +- src/allmydata/test/test_dirnode.py | 2 +- src/allmydata/test/test_storage_client.py | 6 +----- src/allmydata/test/web/test_web.py | 4 ++-- 4 files changed, 5 insertions(+), 9 deletions(-) diff --git a/src/allmydata/test/no_network.py b/src/allmydata/test/no_network.py index ba497f81a..b9c9ecaeb 100644 --- a/src/allmydata/test/no_network.py +++ b/src/allmydata/test/no_network.py @@ -261,7 +261,7 @@ def create_no_network_client(basedir): return defer.succeed(client) -class _NoNetworkClient(_Client): # type: ignore # Cannot determine consistent MRO order +class _NoNetworkClient(_Client): # type: ignore # tahoe-lafs/ticket/3573 """ Overrides all _Client networking functionality to do nothing. """ diff --git a/src/allmydata/test/test_dirnode.py b/src/allmydata/test/test_dirnode.py index 68fcdd54b..6866bc88e 100644 --- a/src/allmydata/test/test_dirnode.py +++ b/src/allmydata/test/test_dirnode.py @@ -1587,7 +1587,7 @@ class FakeNodeMaker(NodeMaker): def create_mutable_file(self, contents="", keysize=None, version=None): return defer.succeed(FakeMutableFile(contents)) -class FakeClient2(_Client): # type: ignore # ambiguous MRO +class FakeClient2(_Client): # type: ignore # tahoe-lafs/ticket/3573 def __init__(self): self.nodemaker = FakeNodeMaker(None, None, None, None, None, diff --git a/src/allmydata/test/test_storage_client.py b/src/allmydata/test/test_storage_client.py index f2be9ad1e..bcbfffa1e 100644 --- a/src/allmydata/test/test_storage_client.py +++ b/src/allmydata/test/test_storage_client.py @@ -91,11 +91,7 @@ from allmydata.interfaces import ( SOME_FURL = b"pb://abcde@nowhere/fake" -# type checks fail with: -# Cannot determine consistent method resolution order (MRO) for "NativeStorageServerWithVersion" -# even though class hierarchy is single-inheritance. Probably `implementer` -# wrappers are affecting the MRO. -class NativeStorageServerWithVersion(NativeStorageServer): # type: ignore +class NativeStorageServerWithVersion(NativeStorageServer): # type: ignore # tahoe-lafs/ticket/3573 def __init__(self, version): # note: these instances won't work for anything other than # get_available_space() because we don't upcall diff --git a/src/allmydata/test/web/test_web.py b/src/allmydata/test/web/test_web.py index a5fbe5a51..ef0446077 100644 --- a/src/allmydata/test/web/test_web.py +++ b/src/allmydata/test/web/test_web.py @@ -189,7 +189,7 @@ class FakeHistory(object): def list_all_helper_statuses(self): return [] -class FakeDisplayableServer(StubServer): # type: ignore # Cannot determine MRO +class FakeDisplayableServer(StubServer): # type: ignore # tahoe-lafs/ticket/3573 def __init__(self, serverid, nickname, connected, last_connect_time, last_loss_time, last_rx_time): StubServer.__init__(self, serverid) @@ -255,7 +255,7 @@ class FakeStorageServer(service.MultiService): def on_status_changed(self, cb): cb(self) -class FakeClient(_Client): # type: ignore # Cannot determine MRO +class FakeClient(_Client): # type: ignore # tahoe-lafs/ticket/3573 def __init__(self): # don't upcall to Client.__init__, since we only want to initialize a # minimal subset From 04ab4dec3be3fdd07f7a85bfe50c3133cef30910 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 26 Dec 2020 13:01:27 -0500 Subject: [PATCH 075/154] Extract function and annotate it to satisfy typechecks for _Config._basedir --- src/allmydata/node.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/allmydata/node.py b/src/allmydata/node.py index 89d5bcb90..fabb255f6 100644 --- a/src/allmydata/node.py +++ b/src/allmydata/node.py @@ -23,6 +23,11 @@ from base64 import b32decode, b32encode from errno import ENOENT, EPERM from warnings import warn +try: + from typing import Union +except ImportError: + pass + import attr # On Python 2 this will be the backported package. @@ -273,6 +278,11 @@ def _error_about_old_config_files(basedir, generated_files): raise e +def ensure_text_and_abspath_expanduser_unicode(basedir): + # type: (Union[bytes, str]) -> str + return abspath_expanduser_unicode(ensure_text(basedir)) + + @attr.s class _Config(object): """ @@ -300,8 +310,8 @@ class _Config(object): config = attr.ib(validator=attr.validators.instance_of(configparser.ConfigParser)) portnum_fname = attr.ib() _basedir = attr.ib( - converter=lambda basedir: abspath_expanduser_unicode(ensure_text(basedir)), - ) + converter=ensure_text_and_abspath_expanduser_unicode, + ) # type: str config_path = attr.ib( validator=attr.validators.optional( attr.validators.instance_of(FilePath), From 3fd46f94009101fdcfff8617ccb335bab255c57f Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 26 Dec 2020 13:02:16 -0500 Subject: [PATCH 076/154] Ignore additional attribute on the function. --- src/allmydata/test/test_node.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/test_node.py b/src/allmydata/test/test_node.py index 1e0f3020c..e44fd5743 100644 --- a/src/allmydata/test/test_node.py +++ b/src/allmydata/test/test_node.py @@ -564,7 +564,7 @@ class TestMissingPorts(unittest.TestCase): config = config_from_string(self.basedir, "portnum", config_data) with self.assertRaises(PortAssignmentRequired): _tub_portlocation(config, None, None) - test_listen_on_zero_with_host.todo = native_str( + test_listen_on_zero_with_host.todo = native_str( # type: ignore "https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3563" ) From cc5a1046d9d936712577717f1ba0409378d279e6 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 26 Dec 2020 13:07:12 -0500 Subject: [PATCH 077/154] Define type for IntroducerService.VERSION, accepting bytes or str as keys for now. --- src/allmydata/introducer/server.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/allmydata/introducer/server.py b/src/allmydata/introducer/server.py index 237c30315..339c5a0ac 100644 --- a/src/allmydata/introducer/server.py +++ b/src/allmydata/introducer/server.py @@ -15,6 +15,12 @@ from past.builtins import long from six import ensure_text import time, os.path, textwrap + +try: + from typing import Any, Dict, Union +except ImportError: + pass + from zope.interface import implementer from twisted.application import service from twisted.internet import defer @@ -147,10 +153,12 @@ class IntroducerService(service.MultiService, Referenceable): name = "introducer" # v1 is the original protocol, added in 1.0 (but only advertised starting # in 1.3), removed in 1.12. v2 is the new signed protocol, added in 1.10 - VERSION = { #"http://allmydata.org/tahoe/protocols/introducer/v1": { }, + # TODO: reconcile bytes/str for keys + VERSION = { + #"http://allmydata.org/tahoe/protocols/introducer/v1": { }, b"http://allmydata.org/tahoe/protocols/introducer/v2": { }, b"application-version": allmydata.__full_version__.encode("utf-8"), - } + } # type: Dict[Union[bytes, str], Any] def __init__(self): service.MultiService.__init__(self) From 854c22e1ca44baa943e36c964c9d06cfa71f8809 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 26 Dec 2020 13:09:57 -0500 Subject: [PATCH 078/154] Use compatible import for urllib.parse.quote. --- src/allmydata/test/web/test_root.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/allmydata/test/web/test_root.py b/src/allmydata/test/web/test_root.py index 0715c8102..2ea418047 100644 --- a/src/allmydata/test/web/test_root.py +++ b/src/allmydata/test/web/test_root.py @@ -1,8 +1,7 @@ import time -from urllib import ( - quote, -) + +from six.moves.urllib.parse import quote from bs4 import ( BeautifulSoup, From 652222116665f6876e476e5a03cf1129fbc1c789 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 26 Dec 2020 13:12:11 -0500 Subject: [PATCH 079/154] Suppress error on SpyHandler interface. --- src/allmydata/test/test_storage_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/test_storage_client.py b/src/allmydata/test/test_storage_client.py index 45a99979f..8500d6bff 100644 --- a/src/allmydata/test/test_storage_client.py +++ b/src/allmydata/test/test_storage_client.py @@ -570,7 +570,7 @@ class SpyEndpoint(object): return d -@implementer(IConnectionHintHandler) +@implementer(IConnectionHintHandler) # type: ignore # warner/foolscap#78 @attr.s class SpyHandler(object): """ From 9780f8bfdcb29120149e4a49254a20fd7576e4ce Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 26 Dec 2020 13:23:28 -0500 Subject: [PATCH 080/154] Add newsfragment --- newsfragments/3399.feature | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/3399.feature diff --git a/newsfragments/3399.feature b/newsfragments/3399.feature new file mode 100644 index 000000000..d30a91679 --- /dev/null +++ b/newsfragments/3399.feature @@ -0,0 +1 @@ +Added 'typechecks' environment for tox running mypy and performing static typechecks. From 4a9d3bde5b53bb7ce9ae38ca03f416f9a4f5aad4 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 26 Dec 2020 13:30:32 -0500 Subject: [PATCH 081/154] Exclude allmydata.scripts.types_ module from PythonTwoRegressions. --- src/allmydata/test/test_python2_regressions.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/allmydata/test/test_python2_regressions.py b/src/allmydata/test/test_python2_regressions.py index 84484f1cf..5c6a654c1 100644 --- a/src/allmydata/test/test_python2_regressions.py +++ b/src/allmydata/test/test_python2_regressions.py @@ -16,6 +16,7 @@ from testtools.matchers import ( BLACKLIST = { "allmydata.test.check_load", "allmydata.windows.registry", + "allmydata.scripts.types_", } From 3951257cd7090188d33f3efabeca263c954912d0 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 27 Dec 2020 16:17:59 -0500 Subject: [PATCH 082/154] Switch to mypy-zope main branch. --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 26e056dc1..706899ebe 100644 --- a/tox.ini +++ b/tox.ini @@ -117,7 +117,7 @@ commands = skip_install = True deps = mypy - git+https://github.com/jaraco/mypy-zope@bugfix/21-InterfaceClass-subclass + git+https://github.com/Shoobx/mypy-zope git+https://github.com/jaraco/foolscap@bugfix/75-use-metaclass commands = mypy src From ce50916ec586d381c9016f441f49bef66e9d020f Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 4 Jan 2021 20:21:43 -0500 Subject: [PATCH 083/154] Add newsfragment --- newsfragments/3536.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3536.minor diff --git a/newsfragments/3536.minor b/newsfragments/3536.minor new file mode 100644 index 000000000..e69de29bb From 88946900b3127dc85cb83320a442c94a4b2f7d52 Mon Sep 17 00:00:00 2001 From: Chad Whitacre Date: Tue, 29 Dec 2020 10:38:07 -0500 Subject: [PATCH 084/154] Port unknown to Python 3 This is covered by test_dirnode/test_grid --- newsfragments/3576.minor | 0 src/allmydata/unknown.py | 10 ++++++++++ src/allmydata/util/_python3.py | 1 + 3 files changed, 11 insertions(+) create mode 100644 newsfragments/3576.minor diff --git a/newsfragments/3576.minor b/newsfragments/3576.minor new file mode 100644 index 000000000..e69de29bb diff --git a/src/allmydata/unknown.py b/src/allmydata/unknown.py index f79c88415..060696293 100644 --- a/src/allmydata/unknown.py +++ b/src/allmydata/unknown.py @@ -1,3 +1,13 @@ +"""Ported to Python 3. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from zope.interface import implementer from twisted.internet import defer diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index db2ceed03..650b52c7a 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -82,6 +82,7 @@ PORTED_MODULES = [ "allmydata.storage.shares", "allmydata.test.no_network", "allmydata.test.mutable.util", + "allmydata.unknown", "allmydata.uri", "allmydata.util._python3", "allmydata.util.abbreviate", From 57282d243111043aaa392f8a2edb5daa1c0749ea Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 4 Jan 2021 20:38:44 -0500 Subject: [PATCH 085/154] Include contribution guidelines for real This warning should go away with this commit: WARNING: toctree contains reference to nonexisting document u'.github/CONTRIBUTING' --- docs/contributing.rst | 1 + docs/index.rst | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 docs/contributing.rst diff --git a/docs/contributing.rst b/docs/contributing.rst new file mode 100644 index 000000000..15e1b6432 --- /dev/null +++ b/docs/contributing.rst @@ -0,0 +1 @@ +.. include:: ../.github/CONTRIBUTING.rst diff --git a/docs/index.rst b/docs/index.rst index 3d0a41302..e5db73db0 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -23,7 +23,7 @@ Contents: frontends/download-status known_issues - ../.github/CONTRIBUTING + contributing CODE_OF_CONDUCT servers From 97454242354072619988ead4067df537952b8b08 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 4 Jan 2021 20:44:13 -0500 Subject: [PATCH 086/154] Include release checklist --- docs/index.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/index.rst b/docs/index.rst index e5db73db0..60a3aa5d4 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -25,6 +25,7 @@ Contents: known_issues contributing CODE_OF_CONDUCT + release-checklist servers helper From d0859b11017a7e049df94b509dda123db06828d0 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 4 Jan 2021 21:04:13 -0500 Subject: [PATCH 087/154] Fix indentation in webapi docs Warning was: tahoe-lafs/docs/frontends/webapi.rst:2035: WARNING: Unexpected indentation. --- docs/frontends/webapi.rst | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/frontends/webapi.rst b/docs/frontends/webapi.rst index 99fa44979..e00445fbf 100644 --- a/docs/frontends/webapi.rst +++ b/docs/frontends/webapi.rst @@ -2032,10 +2032,11 @@ potential for surprises when the file store structure is changed. Tahoe-LAFS provides a mutable file store, but the ways that the store can change are limited. The only things that can change are: - * the mapping from child names to child objects inside mutable directories - (by adding a new child, removing an existing child, or changing an - existing child to point to a different object) - * the contents of mutable files + +* the mapping from child names to child objects inside mutable directories + (by adding a new child, removing an existing child, or changing an + existing child to point to a different object) +* the contents of mutable files Obviously if you query for information about the file store and then act to change it (such as by getting a listing of the contents of a mutable From 4e56cc6d379c35b8a2d479b665b35756279be41b Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 4 Jan 2021 21:19:40 -0500 Subject: [PATCH 088/154] Fix nested bullet lists in release checklist doc --- docs/release-checklist.rst | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/docs/release-checklist.rst b/docs/release-checklist.rst index be32aea6c..da692ef61 100644 --- a/docs/release-checklist.rst +++ b/docs/release-checklist.rst @@ -40,23 +40,31 @@ Create Branch and Apply Updates - Create a branch for release-candidates (e.g. `XXXX.release-1.15.0.rc0`) - run `tox -e news` to produce a new NEWS.txt file (this does a commit) - create the news for the release + - newsfragments/.minor - commit it + - manually fix NEWS.txt + - proper title for latest release ("Release 1.15.0" instead of "Release ...post1432") - double-check date (maybe release will be in the future) - spot-check the release notes (these come from the newsfragments files though so don't do heavy editing) - commit these changes + - update "relnotes.txt" + - update all mentions of 1.14.0 -> 1.15.0 - update "previous release" statement and date - summarize major changes - commit it + - update "CREDITS" + - are there any new contributors in this release? - one way: git log release-1.14.0.. | grep Author | sort | uniq - commit it + - update "docs/known_issues.rst" if appropriate - update "docs/INSTALL.rst" references to the new release - Push the branch to github @@ -82,21 +90,32 @@ they will need to evaluate which contributors' signatures they trust. - (all steps above are completed) - sign the release + - git tag -s -u 0xE34E62D06D0E69CFCA4179FFBDE0D31D68666A7A -m "release Tahoe-LAFS-1.15.0rc0" tahoe-lafs-1.15.0rc0 - (replace the key-id above with your own) + - build all code locally - these should all pass: + - tox -e py27,codechecks,docs,integration + - these can fail (ideally they should not of course): + - tox -e deprecations,upcoming-deprecations + - build tarballs + - tox -e tarballs - confirm it at least exists: - ls dist/ | grep 1.15.0rc0 + - inspect and test the tarballs + - install each in a fresh virtualenv - run `tahoe` command + - when satisfied, sign the tarballs: + - gpg --pinentry=loopback --armor --sign dist/tahoe_lafs-1.15.0rc0-py2-none-any.whl - gpg --pinentry=loopback --armor --sign dist/tahoe_lafs-1.15.0rc0.tar.bz2 - gpg --pinentry=loopback --armor --sign dist/tahoe_lafs-1.15.0rc0.tar.gz @@ -129,6 +148,7 @@ need to be uploaded to https://tahoe-lafs.org in `~source/downloads` https://tahoe-lafs.org/downloads/ on the Web. - scp dist/*1.15.0* username@tahoe-lafs.org:/home/source/downloads - the following developers have access to do this: + - exarkun - meejah - warner @@ -139,6 +159,7 @@ uploaded to PyPI as well. - how to do this? - (original guide says only "twine upload dist/*") - the following developers have access to do this: + - warner - exarkun (partial?) - meejah (partial?) From 1063ee1c1f222276341b4596161455f4f9a7668b Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 4 Jan 2021 21:21:43 -0500 Subject: [PATCH 089/154] Fix warning in release checklist doc Fix "WARNING: Inline emphasis start-string without end-string." --- docs/release-checklist.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/release-checklist.rst b/docs/release-checklist.rst index da692ef61..fabb69fc7 100644 --- a/docs/release-checklist.rst +++ b/docs/release-checklist.rst @@ -157,7 +157,7 @@ For the actual release, the tarball and signature files need to be uploaded to PyPI as well. - how to do this? -- (original guide says only "twine upload dist/*") +- (original guide says only `twine upload dist/*`) - the following developers have access to do this: - warner From f1cf9483566ae719d605df7754630e68fdb6f77f Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Fri, 8 Jan 2021 13:45:16 -0500 Subject: [PATCH 090/154] Enable markdown support in Sphinx configuration The upstream Contributor Covenant is a markdown document. Since we prefer to keep things close to the original, enabling markdown support to render that document seems like a good idea. --- docs/conf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 34ddd1bd4..612c324a3 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -28,7 +28,7 @@ import os # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. -extensions = [] +extensions = ['recommonmark'] # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] @@ -36,7 +36,7 @@ templates_path = ['_templates'] # The suffix(es) of source filenames. # You can specify multiple suffix as a list of string: # source_suffix = ['.rst', '.md'] -source_suffix = '.rst' +source_suffix = ['.rst', '.md'] # The encoding of source files. #source_encoding = 'utf-8-sig' From 65926d6e708b967759b1ad50999bb84ff05c13d2 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Fri, 8 Jan 2021 13:48:04 -0500 Subject: [PATCH 091/154] Install recommonmark in tox "docs" environment We're going to need markdown support to render contributor covenant. --- tox.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/tox.ini b/tox.ini index c61331885..d32f90b15 100644 --- a/tox.ini +++ b/tox.ini @@ -211,6 +211,7 @@ commands = deps = sphinx docutils==0.12 + recommonmark # normal install is not needed for docs, and slows things down skip_install = True commands = From f682d946d0ffffaf1f05d1b9157a83faea84ee5d Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Fri, 8 Jan 2021 13:48:23 -0500 Subject: [PATCH 092/154] Rename docs/README.md After enabling markdown extension in Sphinx configuration, there's a warning about docs/README.md not being included in the toc tree. Since docs/README.md is not to be included in the final rendered document, we'll keep it as a .txt document. --- docs/{README.md => README.txt} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename docs/{README.md => README.txt} (100%) diff --git a/docs/README.md b/docs/README.txt similarity index 100% rename from docs/README.md rename to docs/README.txt From 6107e52f96249ff44726da6f19c0eb430f82f030 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 11 Jan 2021 15:26:38 -0500 Subject: [PATCH 093/154] Fix flake. --- integration/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration/util.py b/integration/util.py index 39ec36a38..64f02a446 100644 --- a/integration/util.py +++ b/integration/util.py @@ -5,7 +5,7 @@ from os import mkdir, environ from os.path import exists, join from six.moves import StringIO from functools import partial -from subprocess import check_output, check_call +from subprocess import check_output from twisted.python.filepath import ( FilePath, From 3489e381be45b094c6dfe7b35102d7cf65bebe55 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 12 Jan 2021 11:16:45 -0500 Subject: [PATCH 094/154] Get rid of finalizer which, I suspect, is keeping tests from shutting down on Windows. --- integration/conftest.py | 3 +++ integration/util.py | 10 ++++++---- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/integration/conftest.py b/integration/conftest.py index 4ae22deee..27404d9e8 100644 --- a/integration/conftest.py +++ b/integration/conftest.py @@ -351,6 +351,9 @@ def alice(reactor, temp_dir, introducer_furl, flog_gatherer, storage_nodes, requ reactor, request, temp_dir, introducer_furl, flog_gatherer, "alice", web_port="tcp:9980:interface=localhost", storage=False, + # We're going to kill this ourselves, so no need for finalizer to + # do it: + finalize=False, ) ) await_client_ready(process) diff --git a/integration/util.py b/integration/util.py index 64f02a446..f5f7029d8 100644 --- a/integration/util.py +++ b/integration/util.py @@ -186,7 +186,7 @@ class TahoeProcess(object): return "".format(self._node_dir) -def _run_node(reactor, node_dir, request, magic_text): +def _run_node(reactor, node_dir, request, magic_text, finalize=True): """ Run a tahoe process from its node_dir. @@ -210,7 +210,8 @@ def _run_node(reactor, node_dir, request, magic_text): ) transport.exited = protocol.exited - request.addfinalizer(partial(_cleanup_tahoe_process, transport, protocol.exited)) + if finalize: + request.addfinalizer(partial(_cleanup_tahoe_process, transport, protocol.exited)) # XXX abusing the Deferred; should use .when_magic_seen() pattern @@ -229,7 +230,8 @@ def _create_node(reactor, request, temp_dir, introducer_furl, flog_gatherer, nam magic_text=None, needed=2, happy=3, - total=4): + total=4, + finalize=True): """ Helper to create a single node, run it and return the instance spawnProcess returned (ITransport) @@ -277,7 +279,7 @@ def _create_node(reactor, request, temp_dir, introducer_furl, flog_gatherer, nam d = Deferred() d.callback(None) d.addCallback(lambda _: created_d) - d.addCallback(lambda _: _run_node(reactor, node_dir, request, magic_text)) + d.addCallback(lambda _: _run_node(reactor, node_dir, request, magic_text, finalize=finalize)) return d From 116c59142d46a033cf4d9ac8615a43fee78cf022 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 12 Jan 2021 11:26:43 -0500 Subject: [PATCH 095/154] Port to Python 3. --- src/allmydata/test/web/test_logs.py | 4 ++++ src/allmydata/util/_python3.py | 1 + 2 files changed, 5 insertions(+) diff --git a/src/allmydata/test/web/test_logs.py b/src/allmydata/test/web/test_logs.py index 4895ed6f0..ca8e5b918 100644 --- a/src/allmydata/test/web/test_logs.py +++ b/src/allmydata/test/web/test_logs.py @@ -9,6 +9,10 @@ from __future__ import ( division, ) +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + from testtools.matchers import ( Equals, ) diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 14db70735..eca06058b 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -186,6 +186,7 @@ PORTED_TEST_MODULES = [ "allmydata.test.test_util", "allmydata.test.web.test_common", "allmydata.test.web.test_grid", + "allmydata.test.web.test_logs", "allmydata.test.web.test_status", "allmydata.test.web.test_util", "allmydata.test.web.test_webish", From d99c94753c1e222e0e5cb9eca79369f5f955f123 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 12 Jan 2021 11:38:37 -0500 Subject: [PATCH 096/154] On Python 3 we need to make sure bytes get written to the websocket. --- src/allmydata/web/logs.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/allmydata/web/logs.py b/src/allmydata/web/logs.py index 0ba8b17e9..896dce418 100644 --- a/src/allmydata/web/logs.py +++ b/src/allmydata/web/logs.py @@ -5,6 +5,8 @@ from __future__ import ( division, ) +from future.builtins import str + import json from autobahn.twisted.resource import WebSocketResource @@ -49,7 +51,11 @@ class TokenAuthenticatedWebSocketServerProtocol(WebSocketServerProtocol): """ # probably want a try/except around here? what do we do if # transmission fails or anything else bad happens? - self.sendMessage(json.dumps(message)) + encoded = json.dumps(message) + if isinstance(encoded, str): + # On Python 3 dumps() returns Unicode... + encoded = encoded.encode("utf-8") + self.sendMessage(encoded) def onOpen(self): """ From c2d69c53096fe1bd5894fd0cd198a475a2d592d5 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 12 Jan 2021 11:41:19 -0500 Subject: [PATCH 097/154] Merge all log tests into one test module. --- src/allmydata/test/test_websocket_logs.py | 54 ---------------------- src/allmydata/test/web/test_logs.py | 56 +++++++++++++++++++++++ 2 files changed, 56 insertions(+), 54 deletions(-) delete mode 100644 src/allmydata/test/test_websocket_logs.py diff --git a/src/allmydata/test/test_websocket_logs.py b/src/allmydata/test/test_websocket_logs.py deleted file mode 100644 index e666a4902..000000000 --- a/src/allmydata/test/test_websocket_logs.py +++ /dev/null @@ -1,54 +0,0 @@ -import json - -from twisted.trial import unittest -from twisted.internet.defer import inlineCallbacks - -from eliot import log_call - -from autobahn.twisted.testing import create_memory_agent, MemoryReactorClockResolver, create_pumper - -from allmydata.web.logs import TokenAuthenticatedWebSocketServerProtocol - - -class TestStreamingLogs(unittest.TestCase): - """ - Test websocket streaming of logs - """ - - def setUp(self): - self.reactor = MemoryReactorClockResolver() - self.pumper = create_pumper() - self.agent = create_memory_agent(self.reactor, self.pumper, TokenAuthenticatedWebSocketServerProtocol) - return self.pumper.start() - - def tearDown(self): - return self.pumper.stop() - - @inlineCallbacks - def test_one_log(self): - """ - write a single Eliot log and see it streamed via websocket - """ - - proto = yield self.agent.open( - transport_config=u"ws://localhost:1234/ws", - options={}, - ) - - messages = [] - def got_message(msg, is_binary=False): - messages.append(json.loads(msg)) - proto.on("message", got_message) - - @log_call(action_type=u"test:cli:some-exciting-action") - def do_a_thing(): - pass - - do_a_thing() - - proto.transport.loseConnection() - yield proto.is_closed - - self.assertEqual(len(messages), 2) - self.assertEqual("started", messages[0]["action_status"]) - self.assertEqual("succeeded", messages[1]["action_status"]) diff --git a/src/allmydata/test/web/test_logs.py b/src/allmydata/test/web/test_logs.py index ca8e5b918..5d697f910 100644 --- a/src/allmydata/test/web/test_logs.py +++ b/src/allmydata/test/web/test_logs.py @@ -1,5 +1,7 @@ """ Tests for ``allmydata.web.logs``. + +Ported to Python 3. """ from __future__ import ( @@ -13,6 +15,15 @@ from future.utils import PY2 if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 +import json + +from twisted.trial import unittest +from twisted.internet.defer import inlineCallbacks + +from eliot import log_call + +from autobahn.twisted.testing import create_memory_agent, MemoryReactorClockResolver, create_pumper + from testtools.matchers import ( Equals, ) @@ -41,6 +52,7 @@ from ..common import ( from ...web.logs import ( create_log_resources, + TokenAuthenticatedWebSocketServerProtocol, ) class StreamingEliotLogsTests(SyncTestCase): @@ -61,3 +73,47 @@ class StreamingEliotLogsTests(SyncTestCase): self.client.get(b"http:///v1"), succeeded(has_response_code(Equals(OK))), ) + + +class TestStreamingLogs(unittest.TestCase): + """ + Test websocket streaming of logs + """ + + def setUp(self): + self.reactor = MemoryReactorClockResolver() + self.pumper = create_pumper() + self.agent = create_memory_agent(self.reactor, self.pumper, TokenAuthenticatedWebSocketServerProtocol) + return self.pumper.start() + + def tearDown(self): + return self.pumper.stop() + + @inlineCallbacks + def test_one_log(self): + """ + write a single Eliot log and see it streamed via websocket + """ + + proto = yield self.agent.open( + transport_config=u"ws://localhost:1234/ws", + options={}, + ) + + messages = [] + def got_message(msg, is_binary=False): + messages.append(json.loads(msg)) + proto.on("message", got_message) + + @log_call(action_type=u"test:cli:some-exciting-action") + def do_a_thing(): + pass + + do_a_thing() + + proto.transport.loseConnection() + yield proto.is_closed + + self.assertEqual(len(messages), 2) + self.assertEqual("started", messages[0]["action_status"]) + self.assertEqual("succeeded", messages[1]["action_status"]) From 7e5e3291381ddf03625375e718658566da009e65 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 12 Jan 2021 11:44:27 -0500 Subject: [PATCH 098/154] Port to Python 3. --- src/allmydata/util/_python3.py | 1 + src/allmydata/web/logs.py | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index eca06058b..9917be84c 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -113,6 +113,7 @@ PORTED_MODULES = [ "allmydata.util.spans", "allmydata.util.statistics", "allmydata.util.time_format", + "allmydata.web.logs", "allmydata.webish", ] diff --git a/src/allmydata/web/logs.py b/src/allmydata/web/logs.py index 896dce418..6f15a3ca9 100644 --- a/src/allmydata/web/logs.py +++ b/src/allmydata/web/logs.py @@ -1,3 +1,6 @@ +""" +Ported to Python 3. +""" from __future__ import ( print_function, unicode_literals, @@ -5,8 +8,6 @@ from __future__ import ( division, ) -from future.builtins import str - import json from autobahn.twisted.resource import WebSocketResource From b74ec6919dc695e661e2523ef8c528535e13332e Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 12 Jan 2021 13:24:42 -0500 Subject: [PATCH 099/154] Don't blow up just because irrelevant cleanup complains. --- integration/conftest.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/integration/conftest.py b/integration/conftest.py index 27404d9e8..1679bd9f9 100644 --- a/integration/conftest.py +++ b/integration/conftest.py @@ -536,7 +536,13 @@ def tor_network(reactor, temp_dir, chutney, request): path=join(chutney_dir), env=env, ) - pytest_twisted.blockon(proto.done) + try: + pytest_twisted.blockon(proto.done) + except ProcessTerminated: + # If this doesn't exit cleanly, that's fine, that shouldn't fail + # the test suite. + pass + request.addfinalizer(cleanup) return chut From dfcd75f20de629723b091c9d5b8ca2aa1b194718 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 12 Jan 2021 13:58:28 -0500 Subject: [PATCH 100/154] Infinite blocking is bad. --- integration/conftest.py | 9 +++++---- integration/util.py | 9 ++++++++- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/integration/conftest.py b/integration/conftest.py index 1679bd9f9..533cbdb67 100644 --- a/integration/conftest.py +++ b/integration/conftest.py @@ -40,7 +40,8 @@ from util import ( TahoeProcess, cli, _run_node, - generate_ssh_key + generate_ssh_key, + block_with_timeout, ) @@ -156,7 +157,7 @@ def flog_gatherer(reactor, temp_dir, flog_binary, request): ) print("Waiting for flogtool to complete") try: - pytest_twisted.blockon(flog_protocol.done) + block_with_timeout(flog_protocol.done, reactor) except ProcessTerminated as e: print("flogtool exited unexpectedly: {}".format(str(e))) print("Flogtool completed") @@ -297,7 +298,7 @@ log_gatherer.furl = {log_furl} def cleanup(): try: transport.signalProcess('TERM') - pytest_twisted.blockon(protocol.exited) + block_with_timeout(protocol.exited, reactor) except ProcessExitedAlready: pass request.addfinalizer(cleanup) @@ -537,7 +538,7 @@ def tor_network(reactor, temp_dir, chutney, request): env=env, ) try: - pytest_twisted.blockon(proto.done) + block_with_timeout(proto.done, reactor) except ProcessTerminated: # If this doesn't exit cleanly, that's fine, that shouldn't fail # the test suite. diff --git a/integration/util.py b/integration/util.py index f5f7029d8..3d1708bae 100644 --- a/integration/util.py +++ b/integration/util.py @@ -28,6 +28,12 @@ from allmydata import client import pytest_twisted +def block_with_timeout(deferred, reactor, timeout=10): + """Block until Deferred has result, but timeout instead of waiting forever.""" + deferred.addTimeout(timeout, reactor) + return pytest_twisted.blockon(deferred) + + class _ProcessExitedProtocol(ProcessProtocol): """ Internal helper that .callback()s on self.done when the process @@ -126,11 +132,12 @@ def _cleanup_tahoe_process(tahoe_transport, exited): :return: After the process has exited. """ + from twisted.internet import reactor try: print("signaling {} with TERM".format(tahoe_transport.pid)) tahoe_transport.signalProcess('TERM') print("signaled, blocking on exit") - pytest_twisted.blockon(exited) + block_with_timeout(exited, reactor) print("exited, goodbye") except ProcessExitedAlready: pass From bd364feec5c9d178929c37179fdc9066f65d97f1 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 12 Jan 2021 13:22:43 -0500 Subject: [PATCH 101/154] Tests pass on Python 3. --- src/allmydata/test/web/test_introducer.py | 2 +- src/allmydata/web/introweb.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/allmydata/test/web/test_introducer.py b/src/allmydata/test/web/test_introducer.py index 929fba507..43f4d5934 100644 --- a/src/allmydata/test/web/test_introducer.py +++ b/src/allmydata/test/web/test_introducer.py @@ -213,7 +213,7 @@ class IntroducerRootTests(unittest.TestCase): resource = IntroducerRoot(introducer_node) response = json.loads( self.successResultOf( - render(resource, {"t": [b"json"]}), + render(resource, {b"t": [b"json"]}), ), ) self.assertEqual( diff --git a/src/allmydata/web/introweb.py b/src/allmydata/web/introweb.py index 6ec558e82..280d6cc26 100644 --- a/src/allmydata/web/introweb.py +++ b/src/allmydata/web/introweb.py @@ -26,10 +26,10 @@ class IntroducerRoot(MultiFormatResource): self.introducer_node = introducer_node self.introducer_service = introducer_node.getServiceNamed("introducer") # necessary as a root Resource - self.putChild("", self) + self.putChild(b"", self) static_dir = resource_filename("allmydata.web", "static") for filen in os.listdir(static_dir): - self.putChild(filen, static.File(os.path.join(static_dir, filen))) + self.putChild(filen.encode("utf-8"), static.File(os.path.join(static_dir, filen))) def _create_element(self): """ @@ -66,7 +66,7 @@ class IntroducerRoot(MultiFormatResource): announcement_summary[service_name] += 1 res[u"announcement_summary"] = announcement_summary - return json.dumps(res, indent=1) + b"\n" + return (json.dumps(res, indent=1) + "\n").encode("utf-8") class IntroducerRootElement(Element): From 8c41f60fdb2b6017f83497ed7a877f955fd7f673 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 12 Jan 2021 14:17:38 -0500 Subject: [PATCH 102/154] Port to Python 3. --- src/allmydata/test/web/test_introducer.py | 12 ++++++++++++ src/allmydata/util/_python3.py | 1 + 2 files changed, 13 insertions(+) diff --git a/src/allmydata/test/web/test_introducer.py b/src/allmydata/test/web/test_introducer.py index 43f4d5934..08d95bda9 100644 --- a/src/allmydata/test/web/test_introducer.py +++ b/src/allmydata/test/web/test_introducer.py @@ -1,3 +1,15 @@ +""" +Ported to Python 3. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + import json from os.path import join diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 9917be84c..63aa5bb0a 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -187,6 +187,7 @@ PORTED_TEST_MODULES = [ "allmydata.test.test_util", "allmydata.test.web.test_common", "allmydata.test.web.test_grid", + "allmydata.test.web.test_introducer", "allmydata.test.web.test_logs", "allmydata.test.web.test_status", "allmydata.test.web.test_util", From c076e1ee2646758a65c42100265e4ddb80e269ef Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 12 Jan 2021 14:24:11 -0500 Subject: [PATCH 103/154] Just fix all the putChild. --- src/allmydata/web/root.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/src/allmydata/web/root.py b/src/allmydata/web/root.py index fdc72ab71..f6316bff5 100644 --- a/src/allmydata/web/root.py +++ b/src/allmydata/web/root.py @@ -1,3 +1,5 @@ +from past.builtins import unicode + import os import time import urllib @@ -227,26 +229,25 @@ class Root(MultiFormatResource): self._client = client self._now_fn = now_fn - # Children need to be bytes; for now just doing these to make specific - # tests pass on Python 3, but eventually will do all them when this - # module is ported to Python 3 (if not earlier). self.putChild(b"uri", URIHandler(client)) - self.putChild("cap", URIHandler(client)) + self.putChild(b"cap", URIHandler(client)) # Handler for everything beneath "/private", an area of the resource # hierarchy which is only accessible with the private per-node API # auth token. - self.putChild("private", create_private_tree(client.get_auth_token)) + self.putChild(b"private", create_private_tree(client.get_auth_token)) - self.putChild("file", FileHandler(client)) - self.putChild("named", FileHandler(client)) - self.putChild("status", status.Status(client.get_history())) - self.putChild("statistics", status.Statistics(client.stats_provider)) + self.putChild(b"file", FileHandler(client)) + self.putChild(b"named", FileHandler(client)) + self.putChild(b"status", status.Status(client.get_history())) + self.putChild(b"statistics", status.Statistics(client.stats_provider)) static_dir = resource_filename("allmydata.web", "static") for filen in os.listdir(static_dir): + if isinstance(filen, unicode): + filen = filen.encode("utf-8") self.putChild(filen, static.File(os.path.join(static_dir, filen))) - self.putChild("report_incident", IncidentReporter()) + self.putChild(b"report_incident", IncidentReporter()) @exception_to_child def getChild(self, path, request): From 4940da47da1cd98ae9e0de3b5b80d11a8c2f573a Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 12 Jan 2021 14:24:17 -0500 Subject: [PATCH 104/154] Tests pass on Python 3. --- src/allmydata/test/web/test_private.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/web/test_private.py b/src/allmydata/test/web/test_private.py index 27ddbcf78..583f629f5 100644 --- a/src/allmydata/test/web/test_private.py +++ b/src/allmydata/test/web/test_private.py @@ -9,6 +9,8 @@ from __future__ import ( division, ) +from future.builtins import str + from testtools.matchers import ( Equals, ) @@ -56,6 +58,7 @@ class PrivacyTests(SyncTestCase): return super(PrivacyTests, self).setUp() def _authorization(self, scheme, value): + value = str(value, "utf-8") return Headers({ u"authorization": [u"{} {}".format(scheme, value)], }) @@ -90,7 +93,7 @@ class PrivacyTests(SyncTestCase): self.assertThat( self.client.head( b"http:///foo/bar", - headers=self._authorization(SCHEME, u"foo bar"), + headers=self._authorization(str(SCHEME, "utf-8"), b"foo bar"), ), succeeded(has_response_code(Equals(UNAUTHORIZED))), ) @@ -103,7 +106,7 @@ class PrivacyTests(SyncTestCase): self.assertThat( self.client.head( b"http:///foo/bar", - headers=self._authorization(SCHEME, self.token), + headers=self._authorization(str(SCHEME, "utf-8"), self.token), ), # It's a made up URL so we don't get a 200, either, but a 404. succeeded(has_response_code(Equals(NOT_FOUND))), From 03fb936716689447edb676c85044cabd79833f3d Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 12 Jan 2021 14:25:16 -0500 Subject: [PATCH 105/154] Port to Python 3. --- src/allmydata/test/web/test_private.py | 6 ++++++ src/allmydata/util/_python3.py | 1 + 2 files changed, 7 insertions(+) diff --git a/src/allmydata/test/web/test_private.py b/src/allmydata/test/web/test_private.py index 583f629f5..293796b1c 100644 --- a/src/allmydata/test/web/test_private.py +++ b/src/allmydata/test/web/test_private.py @@ -1,5 +1,7 @@ """ Tests for ``allmydata.web.private``. + +Ported to Python 3. """ from __future__ import ( @@ -9,6 +11,10 @@ from __future__ import ( division, ) +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + from future.builtins import str from testtools.matchers import ( diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 63aa5bb0a..7cd80335c 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -189,6 +189,7 @@ PORTED_TEST_MODULES = [ "allmydata.test.web.test_grid", "allmydata.test.web.test_introducer", "allmydata.test.web.test_logs", + "allmydata.test.web.test_private", "allmydata.test.web.test_status", "allmydata.test.web.test_util", "allmydata.test.web.test_webish", From 7a3e9ab43e520a531ab63178a06a1fe0160d6aed Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 12 Jan 2021 14:39:20 -0500 Subject: [PATCH 106/154] Tests pass on Python 3. --- src/allmydata/test/web/test_root.py | 8 ++++---- src/allmydata/web/root.py | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/allmydata/test/web/test_root.py b/src/allmydata/test/web/test_root.py index 0715c8102..497440d1c 100644 --- a/src/allmydata/test/web/test_root.py +++ b/src/allmydata/test/web/test_root.py @@ -1,6 +1,6 @@ import time -from urllib import ( +from urllib.parse import ( quote, ) @@ -77,7 +77,7 @@ class RenderSlashUri(unittest.TestCase): ) self.assertEqual( response_body, - "Invalid capability", + b"Invalid capability", ) @@ -92,7 +92,7 @@ class RenderServiceRow(unittest.TestCase): ann = {"anonymous-storage-FURL": "pb://w2hqnbaa25yw4qgcvghl5psa3srpfgw3@tcp:127.0.0.1:51309/vucto2z4fxment3vfxbqecblbf6zyp6x", "permutation-seed-base32": "w2hqnbaa25yw4qgcvghl5psa3srpfgw3", } - srv = NativeStorageServer("server_id", ann, None, {}, EMPTY_CLIENT_CONFIG) + srv = NativeStorageServer(b"server_id", ann, None, {}, EMPTY_CLIENT_CONFIG) srv.get_connection_status = lambda: ConnectionStatus(False, "summary", {}, 0, 0) class FakeClient(_Client): @@ -103,7 +103,7 @@ class RenderServiceRow(unittest.TestCase): tub_maker=None, node_config=EMPTY_CLIENT_CONFIG, ) - self.storage_broker.test_add_server("test-srv", srv) + self.storage_broker.test_add_server(b"test-srv", srv) root = RootElement(FakeClient(), time.time) req = DummyRequest(b"") diff --git a/src/allmydata/web/root.py b/src/allmydata/web/root.py index f6316bff5..5829da51e 100644 --- a/src/allmydata/web/root.py +++ b/src/allmydata/web/root.py @@ -2,7 +2,7 @@ from past.builtins import unicode import os import time -import urllib +from urllib.parse import quote as urlquote from hyperlink import DecodedURL, URL from pkg_resources import resource_filename @@ -83,7 +83,7 @@ class URIHandler(resource.Resource, object): # it seems Nevow was creating absolute URLs including # host/port whereas req.uri is absolute (but lacks host/port) redir_uri = URL.from_text(req.prePathURL().decode('utf8')) - redir_uri = redir_uri.child(urllib.quote(uri_arg).decode('utf8')) + redir_uri = redir_uri.child(urlquote(uri_arg)) # add back all the query args that AREN'T "?uri=" for k, values in req.args.items(): if k != b"uri": From 5d77282784b88dbcb3445d279b9c9d0afd6f6db2 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 12 Jan 2021 14:40:33 -0500 Subject: [PATCH 107/154] Ported to Python 3. --- src/allmydata/test/web/test_root.py | 12 ++++++++++++ src/allmydata/util/_python3.py | 1 + 2 files changed, 13 insertions(+) diff --git a/src/allmydata/test/web/test_root.py b/src/allmydata/test/web/test_root.py index 497440d1c..ca3cc695d 100644 --- a/src/allmydata/test/web/test_root.py +++ b/src/allmydata/test/web/test_root.py @@ -1,3 +1,15 @@ +""" +Ported to Python 3. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + import time from urllib.parse import ( diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 7cd80335c..575a69cb7 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -190,6 +190,7 @@ PORTED_TEST_MODULES = [ "allmydata.test.web.test_introducer", "allmydata.test.web.test_logs", "allmydata.test.web.test_private", + "allmydata.test.web.test_root", "allmydata.test.web.test_status", "allmydata.test.web.test_util", "allmydata.test.web.test_webish", From 6b0849490ad8cbbb23041b16e3850be6e837d30b Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 12 Jan 2021 14:40:46 -0500 Subject: [PATCH 108/154] News file. --- newsfragments/3589.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3589.minor diff --git a/newsfragments/3589.minor b/newsfragments/3589.minor new file mode 100644 index 000000000..e69de29bb From aace119790f92f691c586067f3554a842656c148 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 13 Jan 2021 09:55:54 -0500 Subject: [PATCH 109/154] Fix Python 3 issue with combining bytes and unicode. --- src/allmydata/web/root.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/allmydata/web/root.py b/src/allmydata/web/root.py index 5829da51e..e3d49cd66 100644 --- a/src/allmydata/web/root.py +++ b/src/allmydata/web/root.py @@ -243,9 +243,10 @@ class Root(MultiFormatResource): self.putChild(b"statistics", status.Statistics(client.stats_provider)) static_dir = resource_filename("allmydata.web", "static") for filen in os.listdir(static_dir): + child_path = filen if isinstance(filen, unicode): - filen = filen.encode("utf-8") - self.putChild(filen, static.File(os.path.join(static_dir, filen))) + child_path = filen.encode("utf-8") + self.putChild(child_path, static.File(os.path.join(static_dir, filen))) self.putChild(b"report_incident", IncidentReporter()) From c5669e16e0ba0943e83fd7f3e39a2569eb582743 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 13 Jan 2021 09:56:08 -0500 Subject: [PATCH 110/154] Fix flake. --- src/allmydata/test/web/test_private.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/allmydata/test/web/test_private.py b/src/allmydata/test/web/test_private.py index 293796b1c..b426b4d93 100644 --- a/src/allmydata/test/web/test_private.py +++ b/src/allmydata/test/web/test_private.py @@ -15,8 +15,6 @@ from future.utils import PY2 if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 -from future.builtins import str - from testtools.matchers import ( Equals, ) From 20e90b4b65cbe67a5bda54fb4a1a52fcf2de5a3e Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 13 Jan 2021 10:21:00 -0500 Subject: [PATCH 111/154] Set --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index c61331885..110be7a51 100644 --- a/tox.ini +++ b/tox.ini @@ -77,7 +77,7 @@ setenv = COVERAGE_PROCESS_START=.coveragerc commands = # NOTE: 'run with "py.test --keep-tempdir -s -v integration/" to debug failures' - py.test --coverage -v {posargs:integration} + py.test --timeout=1800 --coverage -v {posargs:integration} coverage combine coverage report From 9ca17d780ebfc972cf5b6b319d5303dab0a3c4ce Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 13 Jan 2021 10:21:06 -0500 Subject: [PATCH 112/154] Add some overall timeout, and timeout on specific test that seems to be the issue somehow. --- integration/test_web.py | 3 +++ setup.py | 1 + 2 files changed, 4 insertions(+) diff --git a/integration/test_web.py b/integration/test_web.py index fe2137ff3..a16bf2e71 100644 --- a/integration/test_web.py +++ b/integration/test_web.py @@ -21,6 +21,8 @@ import requests import html5lib from bs4 import BeautifulSoup +import pytest + def test_index(alice): """ @@ -175,6 +177,7 @@ def test_deep_stats(alice): time.sleep(.5) +@pytest.mark.timeout(60) def test_status(alice): """ confirm we get something sensible from /status and the various sub-types diff --git a/setup.py b/setup.py index 952a921bc..5dc68d367 100644 --- a/setup.py +++ b/setup.py @@ -396,6 +396,7 @@ setup(name="tahoe-lafs", # also set in __init__.py "junitxml", "tenacity", "paramiko", + "pytest-timeout", ] + tor_requires + i2p_requires, "tor": tor_requires, "i2p": i2p_requires, From a2dab7c89fed217603116b97fd7d5ca8d0426823 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 14 Jan 2021 09:40:10 -0500 Subject: [PATCH 113/154] Only do this on Python 3. --- src/allmydata/web/root.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/allmydata/web/root.py b/src/allmydata/web/root.py index e3d49cd66..b1ba501a9 100644 --- a/src/allmydata/web/root.py +++ b/src/allmydata/web/root.py @@ -1,3 +1,4 @@ +from future.utils import PY3 from past.builtins import unicode import os @@ -244,7 +245,7 @@ class Root(MultiFormatResource): static_dir = resource_filename("allmydata.web", "static") for filen in os.listdir(static_dir): child_path = filen - if isinstance(filen, unicode): + if PY3: child_path = filen.encode("utf-8") self.putChild(child_path, static.File(os.path.join(static_dir, filen))) From 42b31a28099ed7f5c732c4bfc6a71db46b303559 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 14 Jan 2021 15:58:18 -0500 Subject: [PATCH 114/154] Fix flake. --- src/allmydata/web/root.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/allmydata/web/root.py b/src/allmydata/web/root.py index b1ba501a9..0ef6b00d2 100644 --- a/src/allmydata/web/root.py +++ b/src/allmydata/web/root.py @@ -1,5 +1,4 @@ from future.utils import PY3 -from past.builtins import unicode import os import time From a01078ddec86e1d193c7940087d6506d376028d8 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 15 Jan 2021 15:00:57 -0500 Subject: [PATCH 115/154] Switch to one of our Docker images for typecheck CI --- .circleci/config.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ff14d6dd3..29b55ad5f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -481,13 +481,14 @@ jobs: typechecks: docker: - <<: *DOCKERHUB_AUTH - image: "jaraco/multipy-tox" + image: "tahoelafsci/ubuntu:18.04-py3" steps: - "checkout" - run: name: "Validate Types" - command: tox -e typechecks + command: | + /tmp/venv/bin/tox -e typechecks build-image: &BUILD_IMAGE # This is a template for a job to build a Docker image that has as much of From c7a4cdb44d801034de1e068f10aa3d8df8a4950f Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 15 Jan 2021 15:25:07 -0500 Subject: [PATCH 116/154] Rely on main branch of foolscap for typechecks. --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 706899ebe..2873bded8 100644 --- a/tox.ini +++ b/tox.ini @@ -118,7 +118,7 @@ skip_install = True deps = mypy git+https://github.com/Shoobx/mypy-zope - git+https://github.com/jaraco/foolscap@bugfix/75-use-metaclass + git+https://github.com/warner/foolscap commands = mypy src From 621de4d882a8a0df0bb555897380dc9e7f111302 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 18 Jan 2021 10:55:18 -0500 Subject: [PATCH 117/154] Add newsfragment --- newsfragments/3591.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3591.minor diff --git a/newsfragments/3591.minor b/newsfragments/3591.minor new file mode 100644 index 000000000..e69de29bb From fa1a8e8371a2a6567901e8b5dcd7259f0e1e7352 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 18 Jan 2021 10:57:31 -0500 Subject: [PATCH 118/154] Upgrade pip used in GitHub Actions From pip 20.1+ onward, "pip cache dir" can be used to find location of pip cache, and this is useful across all three major OSes supported by GitHub Actions. --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fd5049104..50a1a3d2f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -43,7 +43,7 @@ jobs: - name: Install Python packages run: | - pip install --upgrade codecov tox setuptools + pip install --upgrade codecov tox setuptools pip pip list - name: Display tool versions @@ -114,7 +114,7 @@ jobs: - name: Install Python packages run: | - pip install --upgrade tox + pip install --upgrade tox pip pip list - name: Display tool versions @@ -166,7 +166,7 @@ jobs: - name: Install Python packages run: | - pip install --upgrade tox + pip install --upgrade tox pip pip list - name: Display tool versions From 27a122088cdba445465428da6e7680c9a22e6c74 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 18 Jan 2021 11:02:55 -0500 Subject: [PATCH 119/154] Use pip cache on GitHub Actions Using the method outlined in https://github.com/actions/cache/blob/main/examples.md#using-pip-to-get-cache-location --- .github/workflows/ci.yml | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 50a1a3d2f..8ca015e07 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -46,6 +46,19 @@ jobs: pip install --upgrade codecov tox setuptools pip pip list + - name: Get pip cache directory + id: pip-cache + run: | + echo "::set-output name=dir::$(pip cache dir)" + + - name: pip cache + uses: actions/cache@v2 + with: + path: ${{ steps.pip-cache.outputs.dir }} + key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }} + restore-keys: | + ${{ runner.os }}-pip- + - name: Display tool versions run: python misc/build_helpers/show-tool-versions.py @@ -117,6 +130,19 @@ jobs: pip install --upgrade tox pip pip list + - name: Get pip cache directory + id: pip-cache + run: | + echo "::set-output name=dir::$(pip cache dir)" + + - name: pip cache + uses: actions/cache@v2 + with: + path: ${{ steps.pip-cache.outputs.dir }} + key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }} + restore-keys: | + ${{ runner.os }}-pip- + - name: Display tool versions run: python misc/build_helpers/show-tool-versions.py @@ -169,6 +195,19 @@ jobs: pip install --upgrade tox pip pip list + - name: Get pip cache directory + id: pip-cache + run: | + echo "::set-output name=dir::$(pip cache dir)" + + - name: pip cache + uses: actions/cache@v2 + with: + path: ${{ steps.pip-cache.outputs.dir }} + key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }} + restore-keys: | + ${{ runner.os }}-pip- + - name: Display tool versions run: python misc/build_helpers/show-tool-versions.py From 573ab8768b4443a8e41304708040607974a9f4d4 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 18 Jan 2021 11:08:56 -0500 Subject: [PATCH 120/154] Re-title "use pip cache" step in GitHub Actions --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8ca015e07..51cec9e54 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -51,7 +51,7 @@ jobs: run: | echo "::set-output name=dir::$(pip cache dir)" - - name: pip cache + - name: Use pip cache uses: actions/cache@v2 with: path: ${{ steps.pip-cache.outputs.dir }} @@ -135,7 +135,7 @@ jobs: run: | echo "::set-output name=dir::$(pip cache dir)" - - name: pip cache + - name: Use pip cache uses: actions/cache@v2 with: path: ${{ steps.pip-cache.outputs.dir }} @@ -200,7 +200,7 @@ jobs: run: | echo "::set-output name=dir::$(pip cache dir)" - - name: pip cache + - name: Use pip cache uses: actions/cache@v2 with: path: ${{ steps.pip-cache.outputs.dir }} From f731159cd7608925c5713cb4053067ef81c6bf8b Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 18 Jan 2021 13:13:32 -0500 Subject: [PATCH 121/154] Install Python packages after setting up pip cache --- .github/workflows/ci.yml | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 51cec9e54..c3f32c919 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -41,11 +41,6 @@ jobs: with: python-version: ${{ matrix.python-version }} - - name: Install Python packages - run: | - pip install --upgrade codecov tox setuptools pip - pip list - - name: Get pip cache directory id: pip-cache run: | @@ -59,6 +54,11 @@ jobs: restore-keys: | ${{ runner.os }}-pip- + - name: Install Python packages + run: | + pip install --upgrade codecov tox setuptools pip + pip list + - name: Display tool versions run: python misc/build_helpers/show-tool-versions.py @@ -125,11 +125,6 @@ jobs: with: python-version: ${{ matrix.python-version }} - - name: Install Python packages - run: | - pip install --upgrade tox pip - pip list - - name: Get pip cache directory id: pip-cache run: | @@ -143,6 +138,11 @@ jobs: restore-keys: | ${{ runner.os }}-pip- + - name: Install Python packages + run: | + pip install --upgrade tox pip + pip list + - name: Display tool versions run: python misc/build_helpers/show-tool-versions.py @@ -190,11 +190,6 @@ jobs: with: python-version: ${{ matrix.python-version }} - - name: Install Python packages - run: | - pip install --upgrade tox pip - pip list - - name: Get pip cache directory id: pip-cache run: | @@ -208,6 +203,11 @@ jobs: restore-keys: | ${{ runner.os }}-pip- + - name: Install Python packages + run: | + pip install --upgrade tox pip + pip list + - name: Display tool versions run: python misc/build_helpers/show-tool-versions.py From 8bf068f99152aed31b51ab86b6f3edbfd20baf4a Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 18 Jan 2021 13:19:18 -0500 Subject: [PATCH 122/154] What's the pip version on GitHub Actions? There's no need of upgrading pip if GA offers a sufficiently new pip. --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c3f32c919..0135d9ab9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -44,6 +44,7 @@ jobs: - name: Get pip cache directory id: pip-cache run: | + pip --version echo "::set-output name=dir::$(pip cache dir)" - name: Use pip cache From 1f1a30095ea9359a3c3b752bfbc513efa3685dca Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 18 Jan 2021 13:22:04 -0500 Subject: [PATCH 123/154] Get pip version for all three GitHub Actions OSes --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0135d9ab9..1943933b2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -44,7 +44,6 @@ jobs: - name: Get pip cache directory id: pip-cache run: | - pip --version echo "::set-output name=dir::$(pip cache dir)" - name: Use pip cache @@ -194,6 +193,7 @@ jobs: - name: Get pip cache directory id: pip-cache run: | + pip --version echo "::set-output name=dir::$(pip cache dir)" - name: Use pip cache From 2a1a5cb0a5b232487894dafdfa7200a96f1b1d4b Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 18 Jan 2021 13:32:53 -0500 Subject: [PATCH 124/154] GitHub Actions have sufficiently recent pip At the time of writing this commit message, GitHub Actions offers pip v20.3.3 for both ubuntu-latest and windows-latest, and pip v20.3.1 for macos-latest. Those are sufficiently recent pip versions that have "cache dir" sub-command. --- .github/workflows/ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1943933b2..c3f32c919 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -193,7 +193,6 @@ jobs: - name: Get pip cache directory id: pip-cache run: | - pip --version echo "::set-output name=dir::$(pip cache dir)" - name: Use pip cache From adbe23fe7aac3a0f8b55b3135d266a531ec0bcc4 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 18 Jan 2021 13:34:25 -0500 Subject: [PATCH 125/154] Add a note about pip version on GitHub Actions --- .github/workflows/ci.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c3f32c919..8ccf07aba 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -41,6 +41,11 @@ jobs: with: python-version: ${{ matrix.python-version }} + # We need "pip cache dir", which became a thing in pip v20.1+. + # At the time of writing this, GitHub Actions offers pip v20.3.3 + # for both ubuntu-latest and windows-latest, and pip v20.3.1 for + # macos-latest. Those are sufficiently recent pip versions that + # have "cache dir" sub-command. - name: Get pip cache directory id: pip-cache run: | From 99cca0ea8e7223886b5ee9d184dd70a70141a031 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 18 Jan 2021 13:38:09 -0500 Subject: [PATCH 126/154] No need of upgrading pip on GitHub Actions --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8ccf07aba..afb99c67a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -61,7 +61,7 @@ jobs: - name: Install Python packages run: | - pip install --upgrade codecov tox setuptools pip + pip install --upgrade codecov tox setuptools pip list - name: Display tool versions @@ -145,7 +145,7 @@ jobs: - name: Install Python packages run: | - pip install --upgrade tox pip + pip install --upgrade tox pip list - name: Display tool versions @@ -210,7 +210,7 @@ jobs: - name: Install Python packages run: | - pip install --upgrade tox pip + pip install --upgrade tox pip list - name: Display tool versions From 9e4ea0c4910c819284b9fc74fd488d522bcc5d47 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 18 Jan 2021 17:54:17 -0500 Subject: [PATCH 127/154] Use fetch-depth of 0 with GitHub Actions Using a fetch-depth of 0 should have the same effect as as `git fetch --prune --unshallow` after doing a shallow checkout. --- .github/workflows/ci.yml | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index afb99c67a..213a87ba1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,11 +30,12 @@ jobs: with: args: install vcpython27 + # See https://github.com/actions/checkout. A fetch-depth of 0 + # fetches all tags and branches. - name: Check out Tahoe-LAFS sources uses: actions/checkout@v2 - - - name: Fetch all history for all tags and branches - run: git fetch --prune --unshallow + with: + fetch-depth: 0 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v1 @@ -121,9 +122,8 @@ jobs: - name: Check out Tahoe-LAFS sources uses: actions/checkout@v2 - - - name: Fetch all history for all tags and branches - run: git fetch --prune --unshallow + with: + fetch-depth: 0 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v1 @@ -186,9 +186,8 @@ jobs: - name: Check out Tahoe-LAFS sources uses: actions/checkout@v2 - - - name: Fetch all history for all tags and branches - run: git fetch --prune --unshallow + with: + fetch-depth: 0 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v1 From ed92202762f56c7b37a161064dc4613ec7d9d8d7 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Mon, 18 Jan 2021 17:55:07 -0500 Subject: [PATCH 128/154] Updates comments about GitHub cache action --- .github/workflows/ci.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 213a87ba1..ee36833ca 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -42,16 +42,17 @@ jobs: with: python-version: ${{ matrix.python-version }} - # We need "pip cache dir", which became a thing in pip v20.1+. - # At the time of writing this, GitHub Actions offers pip v20.3.3 - # for both ubuntu-latest and windows-latest, and pip v20.3.1 for - # macos-latest. Those are sufficiently recent pip versions that - # have "cache dir" sub-command. + # To use pip caching with GitHub Actions in an OS-independent + # manner, we need `pip cache dir` command, which became + # available since pip v20.1+. At the time of writing this, + # GitHub Actions offers pip v20.3.3 for both ubuntu-latest and + # windows-latest, and pip v20.3.1 for macos-latest. - name: Get pip cache directory id: pip-cache run: | echo "::set-output name=dir::$(pip cache dir)" + # See https://github.com/actions/cache - name: Use pip cache uses: actions/cache@v2 with: From afcae42fd6d3f9df21cee943aeb1a65140d7013a Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 15 Jan 2021 11:52:16 -0500 Subject: [PATCH 129/154] Notice that there's an error on the server, rather than continuing silently. --- integration/test_web.py | 1 + 1 file changed, 1 insertion(+) diff --git a/integration/test_web.py b/integration/test_web.py index fe2137ff3..a322129f8 100644 --- a/integration/test_web.py +++ b/integration/test_web.py @@ -133,6 +133,7 @@ def test_deep_stats(alice): u"file": FILE_CONTENTS, }, ) + resp.raise_for_status() # confirm the file is in the directory resp = requests.get( From 5dd7aa2dfda61156f83e24c37312238a631757c1 Mon Sep 17 00:00:00 2001 From: meejah Date: Tue, 19 Jan 2021 11:25:09 -0700 Subject: [PATCH 130/154] news --- newsfragments/2920.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/2920.minor diff --git a/newsfragments/2920.minor b/newsfragments/2920.minor new file mode 100644 index 000000000..e69de29bb From 3166545509867c158b822794dfc26d716ee8224a Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 19 Jan 2021 13:52:12 -0500 Subject: [PATCH 131/154] Unit test reproducing the bug in the integration test. --- src/allmydata/test/web/test_web.py | 33 ++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/src/allmydata/test/web/test_web.py b/src/allmydata/test/web/test_web.py index ce9a40389..c7d866ad1 100644 --- a/src/allmydata/test/web/test_web.py +++ b/src/allmydata/test/web/test_web.py @@ -4757,6 +4757,39 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi op_url = self.webish_url + "/operations/134?t=status&output=JSON" yield self.assertHTTPError(op_url, 404, "unknown/expired handle '134'") + @inlineCallbacks + def test_upload_file_in_directory(self): + """Create a directory, then upload a file into it. + + Unit test reproducer for https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3590 + """ + def req(method, path, **kwargs): + return treq.request(method, self.webish_url + path, persistent=False, + **kwargs) + + response = yield req("POST", "/uri?format=sdmf&t=mkdir&redirect_to_result=true", + browser_like_redirects=True) + + uri = urllib.unquote(response.request.absoluteURI) + assert 'URI:DIR2:' in uri + dircap = uri[uri.find("URI:DIR2:"):].rstrip('/') + dircap_uri = "/uri/{}".format(urllib.quote(dircap)) + + # POST a file into this directory + FILE_CONTENTS = u"a file in a directory" + + body, headers = self.build_form(t="upload", when_done=".", name="file", + file=FILE_CONTENTS) + response = yield req( + "POST", + dircap_uri, + data=body, + headers=headers, + browser_like_redirects=True + ) + if response.code >= 400: + raise Error(response.code, response=response.content()) + def test_incident(self): d = self.POST("/report_incident", details="eek") def _done(res): From 6979cfa205ee13da0358dd92d0971d78c09ffefc Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 19 Jan 2021 14:28:00 -0500 Subject: [PATCH 132/154] Fix the redirect 'str has no render' bug. --- src/allmydata/web/root.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/web/root.py b/src/allmydata/web/root.py index 0ef6b00d2..2e82e94ec 100644 --- a/src/allmydata/web/root.py +++ b/src/allmydata/web/root.py @@ -11,7 +11,7 @@ from twisted.web import ( resource, static, ) -from twisted.web.util import redirectTo +from twisted.web.util import redirectTo, Redirect from twisted.python.filepath import FilePath from twisted.web.template import ( Element, @@ -155,7 +155,7 @@ class URIHandler(resource.Resource, object): u = u.replace( path=(s for s in u.path if s), # remove empty segments ) - return redirectTo(u.to_uri().to_text().encode('utf8'), req) + return Redirect(u.to_uri().to_text().encode('utf8')) try: node = self.client.create_node_from_uri(name) return directory.make_handler_for(node, self.client) From e91d37e64b50579bd75edf23dd767161b699611d Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 19 Jan 2021 17:13:52 -0500 Subject: [PATCH 133/154] Fix unit test so it's actually testing the real bug. --- src/allmydata/test/web/test_web.py | 22 ++++++++-------------- src/allmydata/web/root.py | 2 +- 2 files changed, 9 insertions(+), 15 deletions(-) diff --git a/src/allmydata/test/web/test_web.py b/src/allmydata/test/web/test_web.py index c7d866ad1..8dcb7d95b 100644 --- a/src/allmydata/test/web/test_web.py +++ b/src/allmydata/test/web/test_web.py @@ -4758,8 +4758,8 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi yield self.assertHTTPError(op_url, 404, "unknown/expired handle '134'") @inlineCallbacks - def test_upload_file_in_directory(self): - """Create a directory, then upload a file into it. + def test_uri_redirect(self): + """URI redirects don't cause failure. Unit test reproducer for https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3590 """ @@ -4767,26 +4767,20 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi return treq.request(method, self.webish_url + path, persistent=False, **kwargs) - response = yield req("POST", "/uri?format=sdmf&t=mkdir&redirect_to_result=true", - browser_like_redirects=True) + response = yield req("POST", "/uri?format=sdmf&t=mkdir&redirect_to_result=true") uri = urllib.unquote(response.request.absoluteURI) assert 'URI:DIR2:' in uri dircap = uri[uri.find("URI:DIR2:"):].rstrip('/') - dircap_uri = "/uri/{}".format(urllib.quote(dircap)) + dircap_uri = "/uri/?uri={}&t=json".format(urllib.quote(dircap)) - # POST a file into this directory - FILE_CONTENTS = u"a file in a directory" - - body, headers = self.build_form(t="upload", when_done=".", name="file", - file=FILE_CONTENTS) response = yield req( - "POST", + "GET", dircap_uri, - data=body, - headers=headers, - browser_like_redirects=True ) + self.assertEqual( + response.request.absoluteURI, + self.webish_url + "/uri/{}?t=json".format(urllib.quote(dircap))) if response.code >= 400: raise Error(response.code, response=response.content()) diff --git a/src/allmydata/web/root.py b/src/allmydata/web/root.py index 2e82e94ec..9fb3ac9d3 100644 --- a/src/allmydata/web/root.py +++ b/src/allmydata/web/root.py @@ -147,7 +147,7 @@ class URIHandler(resource.Resource, object): and creates and appropriate handler (depending on the kind of capability it was passed). """ - # this is in case a URI like "/uri/?cap=" is + # this is in case a URI like "/uri/?uri=" is # passed -- we re-direct to the non-trailing-slash version so # that there is just one valid URI for "uri" resource. if not name: From 7d2aa50894bb59043b5e722425d71726f871eb4c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 19 Jan 2021 17:15:07 -0500 Subject: [PATCH 134/154] when_done is bad, at least here. --- integration/test_web.py | 1 - 1 file changed, 1 deletion(-) diff --git a/integration/test_web.py b/integration/test_web.py index a322129f8..216d80d42 100644 --- a/integration/test_web.py +++ b/integration/test_web.py @@ -127,7 +127,6 @@ def test_deep_stats(alice): dircap_uri, data={ u"t": u"upload", - u"when_done": u".", }, files={ u"file": FILE_CONTENTS, From 53243540cc9ad5310dd9d97de0ca7e1afeed1068 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 19 Jan 2021 17:15:58 -0500 Subject: [PATCH 135/154] News file. --- 3590.bugfix | 1 + 1 file changed, 1 insertion(+) create mode 100644 3590.bugfix diff --git a/3590.bugfix b/3590.bugfix new file mode 100644 index 000000000..aa504a5e3 --- /dev/null +++ b/3590.bugfix @@ -0,0 +1 @@ +Fixed issue where redirecting old-style URIs (/uri/?uri=...) didn't work. \ No newline at end of file From 8be3678cb47f0902a94d2ed1b1d651842b738efd Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 20 Jan 2021 11:22:22 -0500 Subject: [PATCH 136/154] Directly test read_encrypted behavior --- src/allmydata/test/test_upload.py | 69 +++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/src/allmydata/test/test_upload.py b/src/allmydata/test/test_upload.py index 94d7575c3..7e41bfc24 100644 --- a/src/allmydata/test/test_upload.py +++ b/src/allmydata/test/test_upload.py @@ -14,6 +14,17 @@ if PY2: import os, shutil from io import BytesIO +from base64 import ( + b64encode, +) + +from hypothesis import ( + given, +) +from hypothesis.strategies import ( + just, + integers, +) from twisted.trial import unittest from twisted.python.failure import Failure @@ -2029,6 +2040,64 @@ class EncodingParameters(GridTestMixin, unittest.TestCase, SetDEPMixin, f.close() return None + +class EncryptAnUploadableTests(unittest.TestCase): + """ + Tests for ``EncryptAnUploadable``. + """ + def test_same_length(self): + """ + ``EncryptAnUploadable.read_encrypted`` returns ciphertext of the same + length as the underlying plaintext. + """ + plaintext = b"hello world" + uploadable = upload.FileHandle(BytesIO(plaintext), None) + uploadable.set_default_encoding_parameters({ + # These values shouldn't matter. + "k": 3, + "happy": 5, + "n": 10, + "max_segment_size": 128 * 1024, + }) + encrypter = upload.EncryptAnUploadable(uploadable) + ciphertext = b"".join(self.successResultOf(encrypter.read_encrypted(1024, False))) + self.assertEqual(len(ciphertext), len(plaintext)) + + @given(just(b"hello world"), integers(min_value=0, max_value=len(b"hello world"))) + def test_known_result(self, plaintext, split_at): + """ + ``EncryptAnUploadable.read_encrypted`` returns a known-correct ciphertext + string for certain inputs. The ciphertext is independent of the read + sizes. + """ + convergence = b"\x42" * 16 + uploadable = upload.FileHandle(BytesIO(plaintext), convergence) + uploadable.set_default_encoding_parameters({ + # The convergence key is a function of k, n, and max_segment_size + # (among other things). The value for happy doesn't matter + # though. + "k": 3, + "happy": 5, + "n": 10, + "max_segment_size": 128 * 1024, + }) + encrypter = upload.EncryptAnUploadable(uploadable) + def read(n): + return b"".join(self.successResultOf(encrypter.read_encrypted(n, False))) + + # Read the string in one or two pieces to make sure underlying state + # is maintained properly. + first = read(split_at) + second = read(len(plaintext) - split_at) + third = read(1) + ciphertext = first + second + third + + self.assertEqual( + b"Jd2LHCRXozwrEJc=", + b64encode(ciphertext), + ) + + # TODO: # upload with exactly 75 servers (shares_of_happiness) # have a download fail From f75f71cba6e98ac508cf06108523a9d0c1a4842f Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 20 Jan 2021 11:23:35 -0500 Subject: [PATCH 137/154] news fragment --- newsfragments/3594.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3594.minor diff --git a/newsfragments/3594.minor b/newsfragments/3594.minor new file mode 100644 index 000000000..e69de29bb From 932481ad47c650cb00070394829a7cc268fdd00e Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 20 Jan 2021 12:58:03 -0500 Subject: [PATCH 138/154] A helper for doing something repeatedly for a while --- src/allmydata/test/test_deferredutil.py | 55 +++++++++++++++++++++++++ src/allmydata/util/deferredutil.py | 30 ++++++++++++++ 2 files changed, 85 insertions(+) diff --git a/src/allmydata/test/test_deferredutil.py b/src/allmydata/test/test_deferredutil.py index 6ebc93556..2a155089f 100644 --- a/src/allmydata/test/test_deferredutil.py +++ b/src/allmydata/test/test_deferredutil.py @@ -74,3 +74,58 @@ class DeferredUtilTests(unittest.TestCase, deferredutil.WaitForDelayedCallsMixin d = defer.succeed(None) d.addBoth(self.wait_for_delayed_calls) return d + + +class UntilTests(unittest.TestCase): + """ + Tests for ``deferredutil.until``. + """ + def test_exception(self): + """ + If the action raises an exception, the ``Deferred`` returned by ``until`` + fires with a ``Failure``. + """ + self.assertFailure( + deferredutil.until(lambda: 1/0, lambda: True), + ZeroDivisionError, + ) + + def test_stops_on_condition(self): + """ + The action is called repeatedly until ``condition`` returns ``True``. + """ + calls = [] + def action(): + calls.append(None) + + def condition(): + return len(calls) == 3 + + self.assertIs( + self.successResultOf( + deferredutil.until(action, condition), + ), + None, + ) + self.assertEqual(3, len(calls)) + + def test_waits_for_deferred(self): + """ + If the action returns a ``Deferred`` then it is called again when the + ``Deferred`` fires. + """ + counter = [0] + r1 = defer.Deferred() + r2 = defer.Deferred() + results = [r1, r2] + def action(): + counter[0] += 1 + return results.pop(0) + + def condition(): + return False + + deferredutil.until(action, condition) + self.assertEqual([1], counter) + r1.callback(None) + self.assertEqual([2], counter) diff --git a/src/allmydata/util/deferredutil.py b/src/allmydata/util/deferredutil.py index 1d13f61e6..ed2a11ee4 100644 --- a/src/allmydata/util/deferredutil.py +++ b/src/allmydata/util/deferredutil.py @@ -15,7 +15,18 @@ if PY2: import time +try: + from typing import ( + Callable, + Any, + ) +except ImportError: + pass + from foolscap.api import eventually +from eliot.twisted import ( + inline_callbacks, +) from twisted.internet import defer, reactor, error from twisted.python.failure import Failure @@ -201,3 +212,22 @@ class WaitForDelayedCallsMixin(PollMixin): d.addErrback(log.err, "error while waiting for delayed calls") d.addBoth(lambda ign: res) return d + +@inline_callbacks +def until( + action, # type: Callable[[], defer.Deferred[Any]] + condition, # type: Callable[[], bool] +): + # type: (...) -> defer.Deferred[None] + """ + Run a Deferred-returning function until a condition is true. + + :param action: The action to run. + :param condition: The predicate signaling stop. + + :return: A Deferred that fires after the condition signals stop. + """ + while True: + yield action() + if condition(): + break From 12087738d682c051aeeb75a7c90dd78d7b8ebfb0 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 20 Jan 2021 13:54:37 -0500 Subject: [PATCH 139/154] Switch from fireEventually to `until` --- src/allmydata/immutable/upload.py | 102 +++++++++++++++++++++--------- src/allmydata/test/test_upload.py | 27 ++++++++ 2 files changed, 98 insertions(+), 31 deletions(-) diff --git a/src/allmydata/immutable/upload.py b/src/allmydata/immutable/upload.py index adcdaed10..fe173b46c 100644 --- a/src/allmydata/immutable/upload.py +++ b/src/allmydata/immutable/upload.py @@ -13,19 +13,33 @@ if PY2: from past.builtins import long, unicode from six import ensure_str +try: + from typing import List +except ImportError: + pass + import os, time, weakref, itertools +from functools import ( + partial, +) + +import attr + from zope.interface import implementer from twisted.python import failure from twisted.internet import defer from twisted.application import service -from foolscap.api import Referenceable, Copyable, RemoteCopy, fireEventually +from foolscap.api import Referenceable, Copyable, RemoteCopy from allmydata.crypto import aes from allmydata.util.hashutil import file_renewal_secret_hash, \ file_cancel_secret_hash, bucket_renewal_secret_hash, \ bucket_cancel_secret_hash, plaintext_hasher, \ storage_index_hash, plaintext_segment_hasher, convergence_hasher -from allmydata.util.deferredutil import timeout_call +from allmydata.util.deferredutil import ( + timeout_call, + until, +) from allmydata import hashtree, uri from allmydata.storage.server import si_b2a from allmydata.immutable import encode @@ -900,13 +914,41 @@ class Tahoe2ServerSelector(log.PrefixingLogMixin): raise UploadUnhappinessError(msg) +@attr.s +class _Accum(object): + """ + Accumulate up to some known amount of ciphertext. + + :ivar remaining: The number of bytes still expected. + :ivar ciphertext: The bytes accumulated so far. + """ + remaining = attr.ib(validator=attr.validators.instance_of(int)) # type: int + ciphertext = attr.ib(default=attr.Factory(list)) # type: List[bytes] + + def extend(self, + size, # type: int + ciphertext, # type: List[bytes] + ): + """ + Accumulate some more ciphertext. + + :param size: The amount of data the new ciphertext represents towards + the goal. This may be more than the actual size of the given + ciphertext if the source has run out of data. + + :param ciphertext: The new ciphertext to accumulate. + """ + self.remaining -= size + self.ciphertext.extend(ciphertext) + + @implementer(IEncryptedUploadable) class EncryptAnUploadable(object): """This is a wrapper that takes an IUploadable and provides IEncryptedUploadable.""" CHUNKSIZE = 50*1024 - def __init__(self, original, log_parent=None, progress=None): + def __init__(self, original, log_parent=None, progress=None, chunk_size=None): precondition(original.default_params_set, "set_default_encoding_parameters not called on %r before wrapping with EncryptAnUploadable" % (original,)) self.original = IUploadable(original) @@ -920,6 +962,8 @@ class EncryptAnUploadable(object): self._ciphertext_bytes_read = 0 self._status = None self._progress = progress + if chunk_size is not None: + self.CHUNKSIZE = chunk_size def set_upload_status(self, upload_status): self._status = IUploadStatus(upload_status) @@ -1026,47 +1070,43 @@ class EncryptAnUploadable(object): # and size d.addCallback(lambda ignored: self.get_size()) d.addCallback(lambda ignored: self._get_encryptor()) - # then fetch and encrypt the plaintext. The unusual structure here - # (passing a Deferred *into* a function) is needed to avoid - # overflowing the stack: Deferreds don't optimize out tail recursion. - # We also pass in a list, to which _read_encrypted will append - # ciphertext. - ciphertext = [] - d2 = defer.Deferred() - d.addCallback(lambda ignored: - self._read_encrypted(length, ciphertext, hash_only, d2)) - d.addCallback(lambda ignored: d2) + + accum = _Accum(length) + action = partial(self._read_encrypted, accum, hash_only) + condition = lambda: accum.remaining == 0 + + d.addCallback(lambda ignored: until(action, condition)) + d.addCallback(lambda ignored: accum.ciphertext) return d - def _read_encrypted(self, remaining, ciphertext, hash_only, fire_when_done): - if not remaining: - fire_when_done.callback(ciphertext) - return None + def _read_encrypted(self, + ciphertext_accum, # type: _Accum + hash_only, # type: bool + ): + # type: (...) -> defer.Deferred + """ + Read the next chunk of plaintext, encrypt it, and extend the accumulator + with the resulting ciphertext. + """ # tolerate large length= values without consuming a lot of RAM by # reading just a chunk (say 50kB) at a time. This only really matters # when hash_only==True (i.e. resuming an interrupted upload), since # that's the case where we will be skipping over a lot of data. - size = min(remaining, self.CHUNKSIZE) - remaining = remaining - size + size = min(ciphertext_accum.remaining, self.CHUNKSIZE) + # read a chunk of plaintext.. d = defer.maybeDeferred(self.original.read, size) - # N.B.: if read() is synchronous, then since everything else is - # actually synchronous too, we'd blow the stack unless we stall for a - # tick. Once you accept a Deferred from IUploadable.read(), you must - # be prepared to have it fire immediately too. - d.addCallback(fireEventually) def _good(plaintext): # and encrypt it.. # o/' over the fields we go, hashing all the way, sHA! sHA! sHA! o/' ct = self._hash_and_encrypt_plaintext(plaintext, hash_only) - ciphertext.extend(ct) - self._read_encrypted(remaining, ciphertext, hash_only, - fire_when_done) - def _err(why): - fire_when_done.errback(why) + # Intentionally tell the accumulator about the expected size, not + # the actual size. If we run out of data we still want remaining + # to drop otherwise it will never reach 0 and the loop will never + # end. + ciphertext_accum.extend(size, ct) d.addCallback(_good) - d.addErrback(_err) - return None + return d def _hash_and_encrypt_plaintext(self, data, hash_only): assert isinstance(data, (tuple, list)), type(data) diff --git a/src/allmydata/test/test_upload.py b/src/allmydata/test/test_upload.py index 7e41bfc24..07ede2074 100644 --- a/src/allmydata/test/test_upload.py +++ b/src/allmydata/test/test_upload.py @@ -2097,6 +2097,33 @@ class EncryptAnUploadableTests(unittest.TestCase): b64encode(ciphertext), ) + def test_large_read(self): + """ + ``EncryptAnUploadable.read_encrypted`` succeeds even when the requested + data length is much larger than the chunk size. + """ + convergence = b"\x42" * 16 + # 4kB of plaintext + plaintext = b"\xde\xad\xbe\xef" * 1024 + uploadable = upload.FileHandle(BytesIO(plaintext), convergence) + uploadable.set_default_encoding_parameters({ + "k": 3, + "happy": 5, + "n": 10, + "max_segment_size": 128 * 1024, + }) + # Make the chunk size very small so we don't have to operate on a huge + # amount of data to exercise the relevant codepath. + encrypter = upload.EncryptAnUploadable(uploadable, chunk_size=1) + d = encrypter.read_encrypted(len(plaintext), False) + ciphertext = self.successResultOf(d) + self.assertEqual( + list(map(len, ciphertext)), + # Chunk size was specified as 1 above so we will get the whole + # plaintext in one byte chunks. + [1] * len(plaintext), + ) + # TODO: # upload with exactly 75 servers (shares_of_happiness) From 9c91261fa6675e2f92890c9cd1229474e49883db Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 20 Jan 2021 13:57:01 -0500 Subject: [PATCH 140/154] news fragment --- newsfragments/3595.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3595.minor diff --git a/newsfragments/3595.minor b/newsfragments/3595.minor new file mode 100644 index 000000000..e69de29bb From 23e52b12377e921bd5cf3b1d15b413cde24aedd3 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 21 Jan 2021 09:58:58 -0500 Subject: [PATCH 141/154] Simplify the unit test. --- src/allmydata/test/web/test_web.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/allmydata/test/web/test_web.py b/src/allmydata/test/web/test_web.py index 8dcb7d95b..2f000b7a1 100644 --- a/src/allmydata/test/web/test_web.py +++ b/src/allmydata/test/web/test_web.py @@ -4767,11 +4767,9 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi return treq.request(method, self.webish_url + path, persistent=False, **kwargs) - response = yield req("POST", "/uri?format=sdmf&t=mkdir&redirect_to_result=true") - - uri = urllib.unquote(response.request.absoluteURI) - assert 'URI:DIR2:' in uri - dircap = uri[uri.find("URI:DIR2:"):].rstrip('/') + response = yield req("POST", "/uri?format=sdmf&t=mkdir") + dircap = yield response.content() + assert dircap.startswith('URI:DIR2:') dircap_uri = "/uri/?uri={}&t=json".format(urllib.quote(dircap)) response = yield req( From db22291660d3a73a8bda617c33299e6e98ca21b7 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 21 Jan 2021 13:54:22 -0500 Subject: [PATCH 142/154] Try to minimally workaround issues causing Windows to block when writing logs. --- integration/test_sftp.py | 7 ++++++- integration/test_web.py | 1 + integration/util.py | 20 ++++++++++++++++++++ 3 files changed, 27 insertions(+), 1 deletion(-) diff --git a/integration/test_sftp.py b/integration/test_sftp.py index f1cf92eab..0b1c600d1 100644 --- a/integration/test_sftp.py +++ b/integration/test_sftp.py @@ -23,7 +23,7 @@ from paramiko.rsakey import RSAKey import pytest -from .util import generate_ssh_key +from .util import generate_ssh_key, run_in_thread def connect_sftp(connect_args={"username": "alice", "password": "password"}): @@ -50,6 +50,7 @@ def connect_sftp(connect_args={"username": "alice", "password": "password"}): return sftp +@run_in_thread def test_bad_account_password_ssh_key(alice, tmpdir): """ Can't login with unknown username, wrong password, or wrong SSH pub key. @@ -79,6 +80,7 @@ def test_bad_account_password_ssh_key(alice, tmpdir): }) +@run_in_thread def test_ssh_key_auth(alice): """It's possible to login authenticating with SSH public key.""" key = RSAKey(filename=join(alice.node_dir, "private", "ssh_client_rsa_key")) @@ -88,6 +90,7 @@ def test_ssh_key_auth(alice): assert sftp.listdir() == [] +@run_in_thread def test_read_write_files(alice): """It's possible to upload and download files.""" sftp = connect_sftp() @@ -102,6 +105,7 @@ def test_read_write_files(alice): f.close() +@run_in_thread def test_directories(alice): """ It's possible to create, list directories, and create and remove files in @@ -135,6 +139,7 @@ def test_directories(alice): assert sftp.listdir() == [] +@run_in_thread def test_rename(alice): """Directories and files can be renamed.""" sftp = connect_sftp() diff --git a/integration/test_web.py b/integration/test_web.py index a16bf2e71..fb45807b3 100644 --- a/integration/test_web.py +++ b/integration/test_web.py @@ -178,6 +178,7 @@ def test_deep_stats(alice): @pytest.mark.timeout(60) +@util.run_in_thread def test_status(alice): """ confirm we get something sensible from /status and the various sub-types diff --git a/integration/util.py b/integration/util.py index 3d1708bae..2b8f587f2 100644 --- a/integration/util.py +++ b/integration/util.py @@ -13,10 +13,12 @@ from twisted.python.filepath import ( from twisted.internet.defer import Deferred, succeed from twisted.internet.protocol import ProcessProtocol from twisted.internet.error import ProcessExitedAlready, ProcessDone +from twisted.internet.threads import deferToThread import requests from paramiko.rsakey import RSAKey +from boltons.funcutils import wraps from allmydata.util.configutil import ( get_config, @@ -525,3 +527,21 @@ def generate_ssh_key(path): key.write_private_key_file(path) with open(path + ".pub", "wb") as f: f.write(b"%s %s" % (key.get_name(), key.get_base64())) + + +def run_in_thread(f): + """Decorator for integration tests that runs code in a thread. + + Because we're using pytest_twisted, tests are expected to return a Deferred + so reactor can run. If the reactor doesn't run, reads from nodes' stdout + and stderr don't happen. eventually the buffers fill up, and the nodes + block when they try to flush logs. + + We can switch to Twisted APIs (treq instead of requests etc.), but + sometimes it's easier or expedient to just have a block test. So this runs + the test in a thread in a way that still lets the reactor run. + """ + @wraps(f) + def test(*args, **kwargs): + return deferToThread(lambda: f(*args, **kwargs)) + return test From 411ee141e98fc2527797bb461a5c08440b479daf Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 21 Jan 2021 13:55:51 -0500 Subject: [PATCH 143/154] Fix location for news fragment. --- 3590.bugfix => newsfragments/3590.bugfix | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename 3590.bugfix => newsfragments/3590.bugfix (100%) diff --git a/3590.bugfix b/newsfragments/3590.bugfix similarity index 100% rename from 3590.bugfix rename to newsfragments/3590.bugfix From 5a0c913f589de968e7543ba7175386454b509be3 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 25 Jan 2021 08:21:39 -0500 Subject: [PATCH 144/154] document the new parameter --- src/allmydata/immutable/upload.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/allmydata/immutable/upload.py b/src/allmydata/immutable/upload.py index fe173b46c..27cc923fd 100644 --- a/src/allmydata/immutable/upload.py +++ b/src/allmydata/immutable/upload.py @@ -949,6 +949,10 @@ class EncryptAnUploadable(object): CHUNKSIZE = 50*1024 def __init__(self, original, log_parent=None, progress=None, chunk_size=None): + """ + :param chunk_size: The number of bytes to read from the uploadable at a + time, or None for some default. + """ precondition(original.default_params_set, "set_default_encoding_parameters not called on %r before wrapping with EncryptAnUploadable" % (original,)) self.original = IUploadable(original) From e0fa2286228a46cd81b82868b56cf096d18d95dc Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 25 Jan 2021 08:23:40 -0500 Subject: [PATCH 145/154] expand partial/lambda into full functions for clarity --- src/allmydata/immutable/upload.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/src/allmydata/immutable/upload.py b/src/allmydata/immutable/upload.py index 27cc923fd..46e01184f 100644 --- a/src/allmydata/immutable/upload.py +++ b/src/allmydata/immutable/upload.py @@ -19,9 +19,6 @@ except ImportError: pass import os, time, weakref, itertools -from functools import ( - partial, -) import attr @@ -1076,8 +1073,18 @@ class EncryptAnUploadable(object): d.addCallback(lambda ignored: self._get_encryptor()) accum = _Accum(length) - action = partial(self._read_encrypted, accum, hash_only) - condition = lambda: accum.remaining == 0 + + def action(): + """ + Read some bytes into the accumulator. + """ + return self._read_encrypted(accum, hash_only) + + def condition(): + """ + Check to see if the accumulator has all the data. + """ + return accum.remaining == 0 d.addCallback(lambda ignored: until(action, condition)) d.addCallback(lambda ignored: accum.ciphertext) From 3b893a56f94dfd57debccc51f2039e0db2e5567f Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 26 Jan 2021 09:55:13 -0500 Subject: [PATCH 146/154] Just rely on global timeout. --- integration/test_web.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/integration/test_web.py b/integration/test_web.py index e89837d30..aab11412f 100644 --- a/integration/test_web.py +++ b/integration/test_web.py @@ -21,8 +21,6 @@ import requests import html5lib from bs4 import BeautifulSoup -import pytest - def test_index(alice): """ @@ -177,7 +175,6 @@ def test_deep_stats(alice): time.sleep(.5) -@pytest.mark.timeout(60) @util.run_in_thread def test_status(alice): """ From 3d2ca566f613a8f238677e71182e025d7a7804f5 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 26 Jan 2021 09:57:02 -0500 Subject: [PATCH 147/154] news fragment --- newsfragments/3599.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3599.minor diff --git a/newsfragments/3599.minor b/newsfragments/3599.minor new file mode 100644 index 000000000..e69de29bb From 0424ba2a487bdc6b3ba6499f6f864e3ffad3e92d Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 26 Jan 2021 09:57:11 -0500 Subject: [PATCH 148/154] Fix indent. --- integration/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration/util.py b/integration/util.py index 2b8f587f2..e8064934f 100644 --- a/integration/util.py +++ b/integration/util.py @@ -267,7 +267,7 @@ def _create_node(reactor, request, temp_dir, introducer_furl, flog_gatherer, nam '--helper', ] if not storage: - args.append('--no-storage') + args.append('--no-storage') args.append(node_dir) _tahoe_runner_optional_coverage(done_proto, reactor, request, args) From e6b3d59501d4b87021491cf03d05bff1516d68b1 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 26 Jan 2021 09:58:08 -0500 Subject: [PATCH 149/154] try to link to the same thing more robustly --- .github/CONTRIBUTING.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/CONTRIBUTING.rst b/.github/CONTRIBUTING.rst index 0c0da9503..b59385aa4 100644 --- a/.github/CONTRIBUTING.rst +++ b/.github/CONTRIBUTING.rst @@ -17,4 +17,4 @@ Examples of contributions include: * `Patch reviews `_ Before authoring or reviewing a patch, -please familiarize yourself with the `Coding Standards `_ and the `Contributor Code of Conduct `_. +please familiarize yourself with the `Coding Standards `_ and the `Contributor Code of Conduct <../docs/CODE_OF_CONDUCT.md>`_. From cc8e613fe3bb1cdec3f84860cfc7c89dee7fb610 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 26 Jan 2021 10:00:50 -0500 Subject: [PATCH 150/154] Rephrase. --- newsfragments/3584.bugfix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/newsfragments/3584.bugfix b/newsfragments/3584.bugfix index 73650f40b..faf57713b 100644 --- a/newsfragments/3584.bugfix +++ b/newsfragments/3584.bugfix @@ -1 +1 @@ -SFTP public key auth likely works better, and SFTP in general was broken in the prerelease. \ No newline at end of file +SFTP public key auth likely works more consistently, and SFTP in general was previously broken. \ No newline at end of file From e7ab792c4c81bd33e989499b304e2fdd212e4715 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 26 Jan 2021 10:06:17 -0500 Subject: [PATCH 151/154] Explain why Paramiko. --- integration/test_sftp.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/integration/test_sftp.py b/integration/test_sftp.py index 0b1c600d1..ed5b37a31 100644 --- a/integration/test_sftp.py +++ b/integration/test_sftp.py @@ -1,6 +1,13 @@ """ It's possible to create/rename/delete files and directories in Tahoe-LAFS using SFTP. + +These tests use Paramiko, rather than Twisted's Conch, because: + + 1. It's a different implementation, so we're not testing Conch against + itself. + + 2. Its API is much simpler to use. """ from __future__ import unicode_literals From 4e89ab2e66f3fb78c92e3060507f1a50a5a74d69 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 26 Jan 2021 10:06:57 -0500 Subject: [PATCH 152/154] Context manager. --- integration/test_sftp.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/integration/test_sftp.py b/integration/test_sftp.py index ed5b37a31..6171c7413 100644 --- a/integration/test_sftp.py +++ b/integration/test_sftp.py @@ -101,15 +101,14 @@ def test_ssh_key_auth(alice): def test_read_write_files(alice): """It's possible to upload and download files.""" sftp = connect_sftp() - f = sftp.file("myfile", "wb") - f.write(b"abc") - f.write(b"def") - f.close() - f = sftp.file("myfile", "rb") - assert f.read(4) == b"abcd" - assert f.read(2) == b"ef" - assert f.read(1) == b"" - f.close() + with sftp.file("myfile", "wb") as f: + f.write(b"abc") + f.write(b"def") + + with sftp.file("myfile", "rb") as f: + assert f.read(4) == b"abcd" + assert f.read(2) == b"ef" + assert f.read(1) == b"" @run_in_thread From 6c04ea74977d745fc4649e871c85bc690e9f9263 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 26 Jan 2021 10:14:14 -0500 Subject: [PATCH 153/154] Explanatory comment is better now. --- integration/util.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/integration/util.py b/integration/util.py index e8064934f..444c8b05e 100644 --- a/integration/util.py +++ b/integration/util.py @@ -532,14 +532,21 @@ def generate_ssh_key(path): def run_in_thread(f): """Decorator for integration tests that runs code in a thread. - Because we're using pytest_twisted, tests are expected to return a Deferred - so reactor can run. If the reactor doesn't run, reads from nodes' stdout - and stderr don't happen. eventually the buffers fill up, and the nodes - block when they try to flush logs. + Because we're using pytest_twisted, tests that rely on the reactor are + expected to return a Deferred and use async APIs so the reactor can run. + + In the case of the integration test suite, it launches nodes in the + background using Twisted APIs. The nodes stdout and stderr is read via + Twisted code. If the reactor doesn't run, reads don't happen, and + eventually the buffers fill up, and the nodes block when they try to flush + logs. We can switch to Twisted APIs (treq instead of requests etc.), but - sometimes it's easier or expedient to just have a block test. So this runs - the test in a thread in a way that still lets the reactor run. + sometimes it's easier or expedient to just have a blocking test. So this + decorator allows you to run the test in a thread, and the reactor can keep + running in the main thread. + + See https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3597 for tracking bug. """ @wraps(f) def test(*args, **kwargs): From d25a0f1ce29d82efd02d1e845f23caa09a2dac93 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 26 Jan 2021 12:40:39 -0500 Subject: [PATCH 154/154] Increase timeout, just to be on the safe side. --- integration/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration/util.py b/integration/util.py index 444c8b05e..256fd68c1 100644 --- a/integration/util.py +++ b/integration/util.py @@ -30,7 +30,7 @@ from allmydata import client import pytest_twisted -def block_with_timeout(deferred, reactor, timeout=10): +def block_with_timeout(deferred, reactor, timeout=120): """Block until Deferred has result, but timeout instead of waiting forever.""" deferred.addTimeout(timeout, reactor) return pytest_twisted.blockon(deferred)