From ee9d9d79848cfcc5e4e49e094db07c25082f9b64 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 11 Sep 2020 15:38:04 -0400 Subject: [PATCH 01/97] Add mypy checks as separate tox environment. --- mypy.ini | 2 ++ tox.ini | 7 +++++++ 2 files changed, 9 insertions(+) create mode 100644 mypy.ini diff --git a/mypy.ini b/mypy.ini new file mode 100644 index 000000000..976ba0294 --- /dev/null +++ b/mypy.ini @@ -0,0 +1,2 @@ +[mypy] +ignore_missing_imports = True diff --git a/tox.ini b/tox.ini index 597270e3a..f5c8f3f42 100644 --- a/tox.ini +++ b/tox.ini @@ -108,6 +108,13 @@ commands = # file. See pyproject.toml for legal values. python -m towncrier.check --pyproject towncrier.pyproject.toml + +[testenv:typechecks] +skip_install = True +deps = mypy +commands = mypy src + + [testenv:draftnews] passenv = TAHOE_LAFS_* PIP_* SUBUNITREPORTER_* USERPROFILE HOMEDRIVE HOMEPATH # see comment in [testenv] about "certifi" From ab54585558f32c92aa9e786df19c42d577e15a44 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 18 Sep 2020 15:49:21 -0400 Subject: [PATCH 02/97] Incorporate mypy-zope to support zope interfaces. --- mypy.ini | 1 + tox.ini | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/mypy.ini b/mypy.ini index 976ba0294..01cbb57a8 100644 --- a/mypy.ini +++ b/mypy.ini @@ -1,2 +1,3 @@ [mypy] ignore_missing_imports = True +plugins=mypy_zope:plugin diff --git a/tox.ini b/tox.ini index f5c8f3f42..1c232b4d7 100644 --- a/tox.ini +++ b/tox.ini @@ -111,7 +111,9 @@ commands = [testenv:typechecks] skip_install = True -deps = mypy +deps = + mypy + mypy-zope commands = mypy src From 6d2d82d7b7342be42d0392ccac0af3ed8882f41c Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Tue, 13 Oct 2020 21:06:07 -0400 Subject: [PATCH 03/97] Use pre-release versions of foolscap and mypy-zope with intended support for RemoteInterface subclasses. --- tox.ini | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 1c232b4d7..a13e840a4 100644 --- a/tox.ini +++ b/tox.ini @@ -113,7 +113,8 @@ commands = skip_install = True deps = mypy - mypy-zope + git+https://github.com/jaraco/mypy-zope@bugfix/21-InterfaceClass-subclass + git+https://github.com/jaraco/foolscap@bugfix/75-use-metaclass commands = mypy src From 4b559ffc3332222c99a86b0f75ea05a01a845321 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 31 Oct 2020 16:06:52 -0400 Subject: [PATCH 04/97] Add typechecks to tox run --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index a13e840a4..0cc0c4ac2 100644 --- a/tox.ini +++ b/tox.ini @@ -7,7 +7,7 @@ twisted = 1 [tox] -envlist = codechecks,py27,py36,pypy27 +envlist = typechecks,codechecks,py27,py36,pypy27 minversion = 2.4 [testenv] From f2ffa78198e756a2337a66463241127c38318509 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 20 Nov 2020 12:32:11 -0500 Subject: [PATCH 05/97] Define type of PollMixin._poll_should_ignore_these_errors --- src/allmydata/util/pollmixin.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/allmydata/util/pollmixin.py b/src/allmydata/util/pollmixin.py index 5d1716853..582bafe86 100644 --- a/src/allmydata/util/pollmixin.py +++ b/src/allmydata/util/pollmixin.py @@ -14,6 +14,12 @@ if PY2: from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import time + +try: + from typing import List +except ImportError: + pass + from twisted.internet import task class TimeoutError(Exception): @@ -23,7 +29,7 @@ class PollComplete(Exception): pass class PollMixin(object): - _poll_should_ignore_these_errors = [] + _poll_should_ignore_these_errors = [] # type: List[Exception] def poll(self, check_f, pollinterval=0.01, timeout=1000): # Return a Deferred, then call check_f periodically until it returns From ce3b775944fdd9070f57de1230c4b2f1c1b8a5b7 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 20 Nov 2020 12:33:41 -0500 Subject: [PATCH 06/97] Suppress typing error in test_python3 --- src/allmydata/test/test_python3.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/test_python3.py b/src/allmydata/test/test_python3.py index 80242f8a2..c1f0e83d6 100644 --- a/src/allmydata/test/test_python3.py +++ b/src/allmydata/test/test_python3.py @@ -44,7 +44,7 @@ class Python3PortingEffortTests(SynchronousTestCase): ), ), ) - test_finished_porting.todo = native_str( + test_finished_porting.todo = native_str( # type: ignore "https://tahoe-lafs.org/trac/tahoe-lafs/milestone/Support%20Python%203 should be completed", ) From d1ea36781a651ece31807c32f366ce6c596b8e70 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 20 Nov 2020 13:12:52 -0500 Subject: [PATCH 07/97] Add type declarations to check_load. --- src/allmydata/test/check_load.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/allmydata/test/check_load.py b/src/allmydata/test/check_load.py index 4058ddf77..21576ea3a 100644 --- a/src/allmydata/test/check_load.py +++ b/src/allmydata/test/check_load.py @@ -37,6 +37,11 @@ a mean of 10kB and a max of 100MB, so filesize=min(int(1.0/random(.0002)),1e8) import os, sys, httplib, binascii import urllib, json, random, time, urlparse +try: + from typing import Dict +except ImportError: + pass + # Python 2 compatibility from future.utils import PY2 if PY2: @@ -49,13 +54,13 @@ if sys.argv[1] == "--stats": DELAY = 10 MAXSAMPLES = 6 totals = [] - last_stats = {} + last_stats = {} # type: Dict[str, float] while True: - stats = {} + stats = {} # type: Dict[str, float] for sf in statsfiles: for line in open(sf, "r").readlines(): - name, value = line.split(":") - value = int(value.strip()) + name, str_value = line.split(":") + value = int(str_value.strip()) if name not in stats: stats[name] = 0 stats[name] += float(value) From 8da82e9ed55d363fb963b95e9b96fac79e6c1bb8 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 20 Nov 2020 13:47:23 -0500 Subject: [PATCH 08/97] Add workaround for Shoobx/mypy-zope#26. --- src/allmydata/web/private.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/allmydata/web/private.py b/src/allmydata/web/private.py index fea058405..a86c869c4 100644 --- a/src/allmydata/web/private.py +++ b/src/allmydata/web/private.py @@ -61,7 +61,11 @@ class IToken(ICredentials): pass -@implementer(IToken) +# Shoobx/mypy-zope#26 +_itoken_impl = implementer(IToken) + + +@_itoken_impl @attr.s class Token(object): proposed_token = attr.ib(type=bytes) From 25cce8b77ebbf6d195f938d39e0c00aa9e6262b9 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Mon, 23 Nov 2020 13:57:37 -0500 Subject: [PATCH 09/97] Suppress typing errors in fileutil, crawler, fixups. --- src/allmydata/storage/crawler.py | 2 +- src/allmydata/util/fileutil.py | 2 +- src/allmydata/windows/fixups.py | 7 ++++++- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/allmydata/storage/crawler.py b/src/allmydata/storage/crawler.py index 24042c38b..f13f7cb99 100644 --- a/src/allmydata/storage/crawler.py +++ b/src/allmydata/storage/crawler.py @@ -19,7 +19,7 @@ import os, time, struct try: import cPickle as pickle except ImportError: - import pickle + import pickle # type: ignore from twisted.internet import reactor from twisted.application import service from allmydata.storage.common import si_b2a diff --git a/src/allmydata/util/fileutil.py b/src/allmydata/util/fileutil.py index ea16c0d6a..e40e06180 100644 --- a/src/allmydata/util/fileutil.py +++ b/src/allmydata/util/fileutil.py @@ -311,7 +311,7 @@ def precondition_abspath(path): _getfullpathname = None try: - from nt import _getfullpathname + from nt import _getfullpathname # type: ignore except ImportError: pass diff --git a/src/allmydata/windows/fixups.py b/src/allmydata/windows/fixups.py index e7f045b95..c5ba3bb57 100644 --- a/src/allmydata/windows/fixups.py +++ b/src/allmydata/windows/fixups.py @@ -217,7 +217,12 @@ def initialize(): # Instead it "mangles" or escapes them using \x7F as an escape character, which we # unescape here. def unmangle(s): - return re.sub(u'\\x7F[0-9a-fA-F]*\\;', lambda m: unichr(int(m.group(0)[1:-1], 16)), s) + return re.sub( + u'\\x7F[0-9a-fA-F]*\\;', + # type ignored for 'unichr' + lambda m: unichr(int(m.group(0)[1:-1], 16)), # type: ignore + s, + ) try: argv = [unmangle(argv_unicode[i]).encode('utf-8') for i in xrange(0, argc.value)] From df31d7db5b23bbc1e12d83f44bf23c9cce316b25 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Mon, 23 Nov 2020 14:05:33 -0500 Subject: [PATCH 10/97] Suppress type error in Node.GENERATED_FILES, apparently unused. --- src/allmydata/node.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/node.py b/src/allmydata/node.py index 0dcd900aa..7622d5bc3 100644 --- a/src/allmydata/node.py +++ b/src/allmydata/node.py @@ -713,7 +713,7 @@ class Node(service.MultiService): """ NODETYPE = "unknown NODETYPE" CERTFILE = "node.pem" - GENERATED_FILES = [] + GENERATED_FILES = [] # type: ignore def __init__(self, config, main_tub, control_tub, i2p_provider, tor_provider): """ From dec6f6d64705b0f103dba3436598f42e96df365c Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Mon, 23 Nov 2020 14:08:23 -0500 Subject: [PATCH 11/97] Remove Interface subclass, as IURI is an interface. Fixes mypy error. --- src/allmydata/interfaces.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/interfaces.py b/src/allmydata/interfaces.py index 49dcf7646..537f6d655 100644 --- a/src/allmydata/interfaces.py +++ b/src/allmydata/interfaces.py @@ -682,7 +682,7 @@ class IURI(Interface): passing into init_from_string.""" -class IVerifierURI(Interface, IURI): +class IVerifierURI(IURI): def init_from_string(uri): """Accept a string (as created by my to_string() method) and populate this instance with its data. I am not normally called directly, From 5f40c562ebb0754c3eabfb0b1dddaeb6d9e4e3a6 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Mon, 23 Nov 2020 14:09:46 -0500 Subject: [PATCH 12/97] Remove self arguments to IProgress, which mypy caught as improper. --- src/allmydata/interfaces.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/interfaces.py b/src/allmydata/interfaces.py index 537f6d655..95b1fdf63 100644 --- a/src/allmydata/interfaces.py +++ b/src/allmydata/interfaces.py @@ -749,7 +749,7 @@ class IProgress(Interface): "Current amount of progress (in percentage)" ) - def set_progress(self, value): + def set_progress(value): """ Sets the current amount of progress. @@ -757,7 +757,7 @@ class IProgress(Interface): set_progress_total. """ - def set_progress_total(self, value): + def set_progress_total(value): """ Sets the total amount of expected progress From adf06889181c41c20db56aece8537a46945ae3ea Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Mon, 23 Nov 2020 14:15:39 -0500 Subject: [PATCH 13/97] Add a non-implementation of encode_proposal to satisfy interface. --- src/allmydata/codec.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/allmydata/codec.py b/src/allmydata/codec.py index a4baab4b6..19345959e 100644 --- a/src/allmydata/codec.py +++ b/src/allmydata/codec.py @@ -57,6 +57,10 @@ class CRSEncoder(object): return defer.succeed((shares, desired_share_ids)) + def encode_proposal(self, data, desired_share_ids=None): + raise NotImplementedError() + + @implementer(ICodecDecoder) class CRSDecoder(object): From 4998c4693fdfd900d0659000888e0e0a59346b8c Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 29 Nov 2020 13:46:52 -0500 Subject: [PATCH 14/97] Ignore type checks on Referenceable objects. Ref warner/foolscap#78. --- src/allmydata/immutable/offloaded.py | 4 ++-- src/allmydata/immutable/upload.py | 2 +- src/allmydata/storage/immutable.py | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/allmydata/immutable/offloaded.py b/src/allmydata/immutable/offloaded.py index d574b980d..4e18ad216 100644 --- a/src/allmydata/immutable/offloaded.py +++ b/src/allmydata/immutable/offloaded.py @@ -141,7 +141,7 @@ class CHKCheckerAndUEBFetcher(object): @implementer(interfaces.RICHKUploadHelper) -class CHKUploadHelper(Referenceable, upload.CHKUploader): +class CHKUploadHelper(Referenceable, upload.CHKUploader): # type: ignore # warner/foolscap#78 """I am the helper-server -side counterpart to AssistedUploader. I handle peer selection, encoding, and share pushing. I read ciphertext from the remote AssistedUploader. @@ -502,7 +502,7 @@ class LocalCiphertextReader(AskUntilSuccessMixin): @implementer(interfaces.RIHelper, interfaces.IStatsProducer) -class Helper(Referenceable): +class Helper(Referenceable): # type: ignore # warner/foolscap#78 """ :ivar dict[bytes, CHKUploadHelper] _active_uploads: For any uploads which have been started but not finished, a mapping from storage index to the diff --git a/src/allmydata/immutable/upload.py b/src/allmydata/immutable/upload.py index e77cbb30b..18f818504 100644 --- a/src/allmydata/immutable/upload.py +++ b/src/allmydata/immutable/upload.py @@ -1423,7 +1423,7 @@ class LiteralUploader(object): return self._status @implementer(RIEncryptedUploadable) -class RemoteEncryptedUploadable(Referenceable): +class RemoteEncryptedUploadable(Referenceable): # type: ignore # warner/foolscap#78 def __init__(self, encrypted_uploadable, upload_status): self._eu = IEncryptedUploadable(encrypted_uploadable) diff --git a/src/allmydata/storage/immutable.py b/src/allmydata/storage/immutable.py index 778c0ddf8..4b60d79f1 100644 --- a/src/allmydata/storage/immutable.py +++ b/src/allmydata/storage/immutable.py @@ -202,7 +202,7 @@ class ShareFile(object): @implementer(RIBucketWriter) -class BucketWriter(Referenceable): +class BucketWriter(Referenceable): # type: ignore # warner/foolscap#78 def __init__(self, ss, incominghome, finalhome, max_size, lease_info, canary): self.ss = ss @@ -301,7 +301,7 @@ class BucketWriter(Referenceable): @implementer(RIBucketReader) -class BucketReader(Referenceable): +class BucketReader(Referenceable): # type: ignore # warner/foolscap#78 def __init__(self, ss, sharefname, storage_index=None, shnum=None): self.ss = ss From 50f81aa25d7ad86c5c238bc2a1d70afce25de03f Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 29 Nov 2020 13:54:39 -0500 Subject: [PATCH 15/97] Update two methods of introducer.client.IntroducerClient to match the interface definition. --- src/allmydata/introducer/client.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/allmydata/introducer/client.py b/src/allmydata/introducer/client.py index 0a6352317..62642d0af 100644 --- a/src/allmydata/introducer/client.py +++ b/src/allmydata/introducer/client.py @@ -157,15 +157,15 @@ class IntroducerClient(service.Service, Referenceable): kwargs["facility"] = "tahoe.introducer.client" return log.msg(*args, **kwargs) - def subscribe_to(self, service_name, cb, *args, **kwargs): - self._local_subscribers.append( (service_name,cb,args,kwargs) ) + def subscribe_to(self, service_name, callback, *args, **kwargs): + self._local_subscribers.append( (service_name,callback,args,kwargs) ) self._subscribed_service_names.add(service_name) self._maybe_subscribe() for index,(ann,key_s,when) in self._inbound_announcements.items(): precondition(isinstance(key_s, str), key_s) servicename = index[0] if servicename == service_name: - eventually(cb, key_s, ann, *args, **kwargs) + eventually(callback, key_s, ann, *args, **kwargs) def _maybe_subscribe(self): if not self._publisher: @@ -198,7 +198,7 @@ class IntroducerClient(service.Service, Referenceable): ann_d.update(ann) return ann_d - def publish(self, service_name, ann, signing_key): + def publish(self, service_name, ann, signing_key=None): # we increment the seqnum every time we publish something new current_seqnum, current_nonce = self._sequencer() From bc3508ce6098d65ae0407047a2e02d3a1bacfee9 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 29 Nov 2020 14:01:05 -0500 Subject: [PATCH 16/97] Ignore type checks on cmp usage (awaiting Python 3 porting) --- src/allmydata/web/status.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/web/status.py b/src/allmydata/web/status.py index 7f6020a99..de15230fd 100644 --- a/src/allmydata/web/status.py +++ b/src/allmydata/web/status.py @@ -1335,7 +1335,7 @@ class Status(MultiFormatResource): active = [s for s in self._get_all_statuses() if s.get_active()] - active.sort(lambda a, b: cmp(a.get_started(), b.get_started())) + active.sort(lambda a, b: cmp(a.get_started(), b.get_started())) # type: ignore # py2 active.reverse() return active @@ -1343,7 +1343,7 @@ class Status(MultiFormatResource): recent = [s for s in self._get_all_statuses() if not s.get_active()] - recent.sort(lambda a, b: cmp(a.get_started(), b.get_started())) + recent.sort(lambda a, b: cmp(a.get_started(), b.get_started())) # type: ignore # py2 recent.reverse() return recent From 6ba7533168eff65454376987258748f99fb1057d Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 29 Nov 2020 14:04:59 -0500 Subject: [PATCH 17/97] Ignore failure on StorageServer.slot_testv_and_readv_and_writev, the implementation of which deviates from the interface spec substantially. --- src/allmydata/storage/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py index 8a8138f26..b7df702d5 100644 --- a/src/allmydata/storage/server.py +++ b/src/allmydata/storage/server.py @@ -581,7 +581,7 @@ class StorageServer(service.MultiService, Referenceable): for share in six.viewvalues(shares): share.add_or_renew_lease(lease_info) - def slot_testv_and_readv_and_writev( + def slot_testv_and_readv_and_writev( # type: ignore # fixme self, storage_index, secrets, From dca0840c350e5b188672e7410a4995b50a1dca60 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 29 Nov 2020 14:08:33 -0500 Subject: [PATCH 18/97] Add stubs for methods demanded by the interface --- src/allmydata/uri.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/allmydata/uri.py b/src/allmydata/uri.py index 2c367cafe..e95c86d96 100644 --- a/src/allmydata/uri.py +++ b/src/allmydata/uri.py @@ -535,6 +535,12 @@ class _DirectoryBaseURI(_BaseURI): def get_storage_index(self): return self._filenode_uri.get_storage_index() + def get_readonly(self): + raise NotImplementedError() + + def is_readonly(self): + raise NotImplementedError() + @implementer(IDirectoryURI) class DirectoryURI(_DirectoryBaseURI): From cc91b7c9edf0d5fc64525b410b50da0748f1180b Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 29 Nov 2020 14:15:24 -0500 Subject: [PATCH 19/97] Declare DirectoryURIVerifier type to allow subclass to override. --- src/allmydata/uri.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/allmydata/uri.py b/src/allmydata/uri.py index e95c86d96..d7f4782cd 100644 --- a/src/allmydata/uri.py +++ b/src/allmydata/uri.py @@ -22,6 +22,11 @@ from past.builtins import unicode, long import re +try: + from typing import Type +except ImportError: + pass + from zope.interface import implementer from twisted.python.components import registerAdapter @@ -707,7 +712,7 @@ class DirectoryURIVerifier(_DirectoryBaseURI): BASE_STRING=b'URI:DIR2-Verifier:' BASE_STRING_RE=re.compile(b'^'+BASE_STRING) - INNER_URI_CLASS=SSKVerifierURI + INNER_URI_CLASS=SSKVerifierURI # type: Type[IVerifierURI] def __init__(self, filenode_uri=None): if filenode_uri: From 7e757d2ec4e201d4e9a1e03be66bd5007eb5c094 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 29 Nov 2020 14:23:59 -0500 Subject: [PATCH 20/97] As _ImmutableFileNodeBase doesn't implement the interface, move the implementer declaration to LiteralFileNode --- src/allmydata/immutable/literal.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/allmydata/immutable/literal.py b/src/allmydata/immutable/literal.py index 68db478f3..6ed5571b9 100644 --- a/src/allmydata/immutable/literal.py +++ b/src/allmydata/immutable/literal.py @@ -19,7 +19,7 @@ from twisted.protocols import basic from allmydata.interfaces import IImmutableFileNode, ICheckable from allmydata.uri import LiteralFileURI -@implementer(IImmutableFileNode, ICheckable) + class _ImmutableFileNodeBase(object): def get_write_uri(self): @@ -56,6 +56,7 @@ class _ImmutableFileNodeBase(object): return not self == other +@implementer(IImmutableFileNode, ICheckable) class LiteralFileNode(_ImmutableFileNodeBase): def __init__(self, filecap): From e9ddcf5911dd75cceea8611dd7a134d7a7eb2fb1 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 29 Nov 2020 14:26:20 -0500 Subject: [PATCH 21/97] Implement set_size as required by the interface --- src/allmydata/immutable/encode.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/allmydata/immutable/encode.py b/src/allmydata/immutable/encode.py index 9351df501..e743ce766 100644 --- a/src/allmydata/immutable/encode.py +++ b/src/allmydata/immutable/encode.py @@ -711,3 +711,6 @@ class Encoder(object): return self.uri_extension_data def get_uri_extension_hash(self): return self.uri_extension_hash + + def set_size(self, size): + raise NotImplementedError() From 1248d65778242c81e914fa6096990c3e3e68fcba Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 29 Nov 2020 14:32:34 -0500 Subject: [PATCH 22/97] Declare types for BasedirOptions. Fixes several errors. --- src/allmydata/scripts/common.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/allmydata/scripts/common.py b/src/allmydata/scripts/common.py index 34266ee72..a633da655 100644 --- a/src/allmydata/scripts/common.py +++ b/src/allmydata/scripts/common.py @@ -4,6 +4,11 @@ import os, sys, urllib, textwrap import codecs from os.path import join +try: + from typing import Optional, Sequence, List +except ImportError: + pass + # Python 2 compatibility from future.utils import PY2 if PY2: @@ -64,7 +69,7 @@ class BasedirOptions(BaseOptions): optParameters = [ ["basedir", "C", None, "Specify which Tahoe base directory should be used. [default: %s]" % quote_local_unicode_path(_default_nodedir)], - ] + ] # type: List[Sequence[Optional[str]]] def parseArgs(self, basedir=None): # This finds the node-directory option correctly even if we are in a subcommand. From e0eb63929a56e46d622fa48b1be159b09cefc031 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 29 Nov 2020 14:35:17 -0500 Subject: [PATCH 23/97] Declare type for BaseOptions.description. Fixes many type errors. --- src/allmydata/scripts/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/scripts/common.py b/src/allmydata/scripts/common.py index a633da655..6d281e07c 100644 --- a/src/allmydata/scripts/common.py +++ b/src/allmydata/scripts/common.py @@ -45,7 +45,7 @@ class BaseOptions(usage.Options): def opt_version(self): raise usage.UsageError("--version not allowed on subcommands") - description = None + description = None # type: Optional[str] description_unwrapped = None def __str__(self): From 1b92da75fa01d94a8486e1a4108c3dcce10458e8 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 29 Nov 2020 14:44:17 -0500 Subject: [PATCH 24/97] Some subclasses use ints, so just go for Any --- src/allmydata/scripts/common.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/scripts/common.py b/src/allmydata/scripts/common.py index 6d281e07c..438b4a7e2 100644 --- a/src/allmydata/scripts/common.py +++ b/src/allmydata/scripts/common.py @@ -5,7 +5,7 @@ import codecs from os.path import join try: - from typing import Optional, Sequence, List + from typing import Optional, Sequence, List, Any except ImportError: pass @@ -69,7 +69,7 @@ class BasedirOptions(BaseOptions): optParameters = [ ["basedir", "C", None, "Specify which Tahoe base directory should be used. [default: %s]" % quote_local_unicode_path(_default_nodedir)], - ] # type: List[Sequence[Optional[str]]] + ] # type: List[Sequence[Any]] def parseArgs(self, basedir=None): # This finds the node-directory option correctly even if we are in a subcommand. From c3a22966e82fd698eab1d62b5416be53b863013b Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 29 Nov 2020 14:52:45 -0500 Subject: [PATCH 25/97] Add stubs for methods demanded by IPeerSelector --- src/allmydata/immutable/upload.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/allmydata/immutable/upload.py b/src/allmydata/immutable/upload.py index 18f818504..171b71eff 100644 --- a/src/allmydata/immutable/upload.py +++ b/src/allmydata/immutable/upload.py @@ -385,6 +385,12 @@ class PeerSelector(object): ) return self.happiness_mappings + def add_peers(self, peerids=None): + raise NotImplementedError + + def confirm_share_allocation(self, peerid, shnum): + raise NotImplementedError + class _QueryStatistics(object): From af172f6bff9b5de980473a02155840e50cdbae91 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 29 Nov 2020 14:55:13 -0500 Subject: [PATCH 26/97] Repeat type declaration from parent to avoid over-constraining this type for subclasses. --- src/allmydata/scripts/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/scripts/common.py b/src/allmydata/scripts/common.py index 438b4a7e2..81511681d 100644 --- a/src/allmydata/scripts/common.py +++ b/src/allmydata/scripts/common.py @@ -106,7 +106,7 @@ class NoDefaultBasedirOptions(BasedirOptions): optParameters = [ ["basedir", "C", None, "Specify which Tahoe base directory should be used."], - ] + ] # type: List[Sequence[Any]] # This is overridden in order to ensure we get a "Wrong number of arguments." # error when more than one argument is given. From 103bec6a1579373df3238c8b5ee09eb72f79718a Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 29 Nov 2020 14:58:28 -0500 Subject: [PATCH 27/97] On MutableFileNode, accept optional 'progress' parameter as declared by the interface. --- src/allmydata/mutable/filenode.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/mutable/filenode.py b/src/allmydata/mutable/filenode.py index 5afc84dec..54e9844d6 100644 --- a/src/allmydata/mutable/filenode.py +++ b/src/allmydata/mutable/filenode.py @@ -564,7 +564,7 @@ class MutableFileNode(object): return d - def upload(self, new_contents, servermap): + def upload(self, new_contents, servermap, progress=None): """ I overwrite the contents of the best recoverable version of this mutable file with new_contents, using servermap instead of From a75454a04f61c3e96b080553fe1f7e412b1d976b Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 29 Nov 2020 15:02:13 -0500 Subject: [PATCH 28/97] Add stub for MutableFileVersion.get_servermap --- src/allmydata/mutable/filenode.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/allmydata/mutable/filenode.py b/src/allmydata/mutable/filenode.py index 54e9844d6..e9cf23fa1 100644 --- a/src/allmydata/mutable/filenode.py +++ b/src/allmydata/mutable/filenode.py @@ -1205,3 +1205,6 @@ class MutableFileVersion(object): self._servermap, mode=mode) return u.update() + + def get_servermap(self): + raise NotImplementedError From 32b77c42394c337fd2bc38f6ff2be29e5acdd001 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 29 Nov 2020 15:06:12 -0500 Subject: [PATCH 29/97] Ignore interface violation in MutableFileVersion.download_to_data --- src/allmydata/mutable/filenode.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/mutable/filenode.py b/src/allmydata/mutable/filenode.py index e9cf23fa1..4613a918b 100644 --- a/src/allmydata/mutable/filenode.py +++ b/src/allmydata/mutable/filenode.py @@ -951,7 +951,7 @@ class MutableFileVersion(object): return self._servermap.size_of_version(self._version) - def download_to_data(self, fetch_privkey=False, progress=None): + def download_to_data(self, fetch_privkey=False, progress=None): # type: ignore # fixme """ I return a Deferred that fires with the contents of this readable object as a byte string. From 646297ddc3f95b9ab88b35d32a83782c8e2f178f Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 29 Nov 2020 15:08:46 -0500 Subject: [PATCH 30/97] Add stub for LocalCiphertextReader.set_upload_status --- src/allmydata/immutable/offloaded.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/allmydata/immutable/offloaded.py b/src/allmydata/immutable/offloaded.py index 4e18ad216..53a1f911a 100644 --- a/src/allmydata/immutable/offloaded.py +++ b/src/allmydata/immutable/offloaded.py @@ -499,6 +499,8 @@ class LocalCiphertextReader(AskUntilSuccessMixin): # ??. I'm not sure if it makes sense to forward the close message. return self.call("close") + def set_upload_status(self, upload_status): + raise NotImplementedError @implementer(interfaces.RIHelper, interfaces.IStatsProducer) From 67f0be8431157f965df50bd82e3098b9be3bbe12 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 29 Nov 2020 15:13:19 -0500 Subject: [PATCH 31/97] Prefer type(None) for better compatibility. --- src/allmydata/frontends/ftpd.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/allmydata/frontends/ftpd.py b/src/allmydata/frontends/ftpd.py index 0b18df85b..af5444969 100644 --- a/src/allmydata/frontends/ftpd.py +++ b/src/allmydata/frontends/ftpd.py @@ -1,7 +1,5 @@ from six import ensure_str -from types import NoneType - from zope.interface import implementer from twisted.application import service, strports from twisted.internet import defer @@ -317,7 +315,7 @@ class Dispatcher(object): class FTPServer(service.MultiService): def __init__(self, client, accountfile, accounturl, ftp_portstr): - precondition(isinstance(accountfile, (unicode, NoneType)), accountfile) + precondition(isinstance(accountfile, (unicode, type(None))), accountfile) service.MultiService.__init__(self) r = Dispatcher(client) From 8b991d3516166b62b7dfe4827ba7b1911de5f680 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 29 Nov 2020 15:16:45 -0500 Subject: [PATCH 32/97] Update DirectoryNode.set_uri to match interface spec. --- src/allmydata/dirnode.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/dirnode.py b/src/allmydata/dirnode.py index 59ebd73ba..6f052c3c7 100644 --- a/src/allmydata/dirnode.py +++ b/src/allmydata/dirnode.py @@ -554,7 +554,7 @@ class DirectoryNode(object): d = self.get_child_and_metadata(childnamex) return d - def set_uri(self, namex, writecap, readcap, metadata=None, overwrite=True): + def set_uri(self, namex, writecap, readcap=None, metadata=None, overwrite=True): precondition(isinstance(writecap, (str,type(None))), writecap) precondition(isinstance(readcap, (str,type(None))), readcap) From 6ea9003436e79163a865a0a30b6051f7811c1bd3 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 29 Nov 2020 15:19:11 -0500 Subject: [PATCH 33/97] Declare MultiFormatResource.formatDefault as optional string for subclass overrides. --- src/allmydata/web/common_py3.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/allmydata/web/common_py3.py b/src/allmydata/web/common_py3.py index 22f235790..080c24c19 100644 --- a/src/allmydata/web/common_py3.py +++ b/src/allmydata/web/common_py3.py @@ -4,6 +4,11 @@ Common utilities that are available from Python 3. Can eventually be merged back into allmydata.web.common. """ +try: + from typing import Optional +except ImportError: + pass + from twisted.web import resource, http from allmydata.util import abbreviate @@ -47,7 +52,7 @@ class MultiFormatResource(resource.Resource, object): format if nothing else is given as the ``formatDefault``. """ formatArgument = "t" - formatDefault = None + formatDefault = None # type: Optional[str] def render(self, req): """ From cb351607d81e21919c3c380c0ac55cd0dba3a5fc Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 29 Nov 2020 15:21:25 -0500 Subject: [PATCH 34/97] Repeat type declaration from parent to avoid over-constraining this type for subclasses. --- src/allmydata/scripts/cli.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/allmydata/scripts/cli.py b/src/allmydata/scripts/cli.py index 379e1d212..96dde65f8 100644 --- a/src/allmydata/scripts/cli.py +++ b/src/allmydata/scripts/cli.py @@ -1,6 +1,12 @@ from __future__ import print_function import os.path, re, fnmatch + +try: + from typing import List, Sequence, Any +except ImportError: + pass + from twisted.python import usage from allmydata.scripts.common import get_aliases, get_default_nodedir, \ DEFAULT_ALIAS, BaseOptions @@ -19,7 +25,7 @@ class FileStoreOptions(BaseOptions): "This overrides the URL found in the --node-directory ."], ["dir-cap", None, None, "Specify which dirnode URI should be used as the 'tahoe' alias."] - ] + ] # type: List[Sequence[Any]] def postOptions(self): self["quiet"] = self.parent["quiet"] From 6b772e7fdc14f473492a61e3e5abf507116077d0 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 29 Nov 2020 15:22:58 -0500 Subject: [PATCH 35/97] Declare type for BaseOptions.description_unwrapped. --- src/allmydata/scripts/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/scripts/common.py b/src/allmydata/scripts/common.py index 81511681d..b501c7e6a 100644 --- a/src/allmydata/scripts/common.py +++ b/src/allmydata/scripts/common.py @@ -46,7 +46,7 @@ class BaseOptions(usage.Options): raise usage.UsageError("--version not allowed on subcommands") description = None # type: Optional[str] - description_unwrapped = None + description_unwrapped = None # type: Optional[str] def __str__(self): width = int(os.environ.get('COLUMNS', '80')) From 41c341a3cc0e2a58847907cc9bee471de4d68519 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 29 Nov 2020 15:24:20 -0500 Subject: [PATCH 36/97] Prefer type(None) for better compatibility. --- src/allmydata/frontends/sftpd.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/allmydata/frontends/sftpd.py b/src/allmydata/frontends/sftpd.py index db914fa45..6ee42d505 100644 --- a/src/allmydata/frontends/sftpd.py +++ b/src/allmydata/frontends/sftpd.py @@ -1,6 +1,5 @@ import six import heapq, traceback, array, stat, struct -from types import NoneType from stat import S_IFREG, S_IFDIR from time import time, strftime, localtime @@ -267,7 +266,7 @@ def _attrs_to_metadata(attrs): def _direntry_for(filenode_or_parent, childname, filenode=None): - precondition(isinstance(childname, (unicode, NoneType)), childname=childname) + precondition(isinstance(childname, (unicode, type(None))), childname=childname) if childname is None: filenode_or_parent = filenode @@ -672,7 +671,7 @@ class GeneralSFTPFile(PrefixingLogMixin): self.log(".open(parent=%r, childname=%r, filenode=%r, metadata=%r)" % (parent, childname, filenode, metadata), level=OPERATIONAL) - precondition(isinstance(childname, (unicode, NoneType)), childname=childname) + precondition(isinstance(childname, (unicode, type(None))), childname=childname) precondition(filenode is None or IFileNode.providedBy(filenode), filenode=filenode) precondition(not self.closed, sftpfile=self) @@ -1194,7 +1193,7 @@ class SFTPUserHandler(ConchUser, PrefixingLogMixin): request = "._sync_heisenfiles(%r, %r, ignore=%r)" % (userpath, direntry, ignore) self.log(request, level=OPERATIONAL) - _assert(isinstance(userpath, str) and isinstance(direntry, (str, NoneType)), + _assert(isinstance(userpath, str) and isinstance(direntry, (str, type(None))), userpath=userpath, direntry=direntry) files = [] @@ -1219,7 +1218,7 @@ class SFTPUserHandler(ConchUser, PrefixingLogMixin): def _remove_heisenfile(self, userpath, parent, childname, file_to_remove): if noisy: self.log("._remove_heisenfile(%r, %r, %r, %r)" % (userpath, parent, childname, file_to_remove), level=NOISY) - _assert(isinstance(userpath, str) and isinstance(childname, (unicode, NoneType)), + _assert(isinstance(userpath, str) and isinstance(childname, (unicode, type(None))), userpath=userpath, childname=childname) direntry = _direntry_for(parent, childname) @@ -1246,7 +1245,7 @@ class SFTPUserHandler(ConchUser, PrefixingLogMixin): (existing_file, userpath, flags, _repr_flags(flags), parent, childname, filenode, metadata), level=NOISY) - _assert((isinstance(userpath, str) and isinstance(childname, (unicode, NoneType)) and + _assert((isinstance(userpath, str) and isinstance(childname, (unicode, type(None))) and (metadata is None or 'no-write' in metadata)), userpath=userpath, childname=childname, metadata=metadata) @@ -1977,7 +1976,7 @@ class Dispatcher(object): class SFTPServer(service.MultiService): def __init__(self, client, accountfile, accounturl, sftp_portstr, pubkey_file, privkey_file): - precondition(isinstance(accountfile, (unicode, NoneType)), accountfile) + precondition(isinstance(accountfile, (unicode, type(None))), accountfile) precondition(isinstance(pubkey_file, unicode), pubkey_file) precondition(isinstance(privkey_file, unicode), privkey_file) service.MultiService.__init__(self) From acbb6b3e93f0a1082e63db42fae20131908afe5e Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 29 Nov 2020 15:48:26 -0500 Subject: [PATCH 37/97] Convert subcommands to tuples instead of lists, as that's what mypy demands for heterogeneous sequences. --- src/allmydata/scripts/admin.py | 9 ++++-- src/allmydata/scripts/cli.py | 39 +++++++++++++------------ src/allmydata/scripts/create_node.py | 13 ++++++--- src/allmydata/scripts/debug.py | 9 ++++-- src/allmydata/scripts/runner.py | 17 +++++++---- src/allmydata/scripts/stats_gatherer.py | 9 ++++-- src/allmydata/scripts/tahoe_invite.py | 7 ++++- src/allmydata/scripts/types_.py | 10 +++++++ 8 files changed, 77 insertions(+), 36 deletions(-) create mode 100644 src/allmydata/scripts/types_.py diff --git a/src/allmydata/scripts/admin.py b/src/allmydata/scripts/admin.py index e472ffd8c..50dde9e43 100644 --- a/src/allmydata/scripts/admin.py +++ b/src/allmydata/scripts/admin.py @@ -1,5 +1,10 @@ from __future__ import print_function +try: + from allmydata.scripts.types_ import SubCommands +except ImportError: + pass + from twisted.python import usage from allmydata.scripts.common import BaseOptions @@ -79,8 +84,8 @@ def do_admin(options): subCommands = [ - ["admin", None, AdminCommand, "admin subcommands: use 'tahoe admin' for a list"], - ] + ("admin", None, AdminCommand, "admin subcommands: use 'tahoe admin' for a list"), + ] # type: SubCommands dispatch = { "admin": do_admin, diff --git a/src/allmydata/scripts/cli.py b/src/allmydata/scripts/cli.py index 96dde65f8..2bb91472f 100644 --- a/src/allmydata/scripts/cli.py +++ b/src/allmydata/scripts/cli.py @@ -4,6 +4,7 @@ import os.path, re, fnmatch try: from typing import List, Sequence, Any + from allmydata.scripts.types_ import SubCommands except ImportError: pass @@ -461,25 +462,25 @@ class DeepCheckOptions(FileStoreOptions): Optionally repair any problems found.""" subCommands = [ - ["mkdir", None, MakeDirectoryOptions, "Create a new directory."], - ["add-alias", None, AddAliasOptions, "Add a new alias cap."], - ["create-alias", None, CreateAliasOptions, "Create a new alias cap."], - ["list-aliases", None, ListAliasesOptions, "List all alias caps."], - ["ls", None, ListOptions, "List a directory."], - ["get", None, GetOptions, "Retrieve a file from the grid."], - ["put", None, PutOptions, "Upload a file into the grid."], - ["cp", None, CpOptions, "Copy one or more files or directories."], - ["unlink", None, UnlinkOptions, "Unlink a file or directory on the grid."], - ["mv", None, MvOptions, "Move a file within the grid."], - ["ln", None, LnOptions, "Make an additional link to an existing file or directory."], - ["backup", None, BackupOptions, "Make target dir look like local dir."], - ["webopen", None, WebopenOptions, "Open a web browser to a grid file or directory."], - ["manifest", None, ManifestOptions, "List all files/directories in a subtree."], - ["stats", None, StatsOptions, "Print statistics about all files/directories in a subtree."], - ["check", None, CheckOptions, "Check a single file or directory."], - ["deep-check", None, DeepCheckOptions, "Check all files/directories reachable from a starting point."], - ["status", None, TahoeStatusCommand, "Various status information."], - ] + ("mkdir", None, MakeDirectoryOptions, "Create a new directory."), + ("add-alias", None, AddAliasOptions, "Add a new alias cap."), + ("create-alias", None, CreateAliasOptions, "Create a new alias cap."), + ("list-aliases", None, ListAliasesOptions, "List all alias caps."), + ("ls", None, ListOptions, "List a directory."), + ("get", None, GetOptions, "Retrieve a file from the grid."), + ("put", None, PutOptions, "Upload a file into the grid."), + ("cp", None, CpOptions, "Copy one or more files or directories."), + ("unlink", None, UnlinkOptions, "Unlink a file or directory on the grid."), + ("mv", None, MvOptions, "Move a file within the grid."), + ("ln", None, LnOptions, "Make an additional link to an existing file or directory."), + ("backup", None, BackupOptions, "Make target dir look like local dir."), + ("webopen", None, WebopenOptions, "Open a web browser to a grid file or directory."), + ("manifest", None, ManifestOptions, "List all files/directories in a subtree."), + ("stats", None, StatsOptions, "Print statistics about all files/directories in a subtree."), + ("check", None, CheckOptions, "Check a single file or directory."), + ("deep-check", None, DeepCheckOptions, "Check all files/directories reachable from a starting point."), + ("status", None, TahoeStatusCommand, "Various status information."), + ] # type: SubCommands def mkdir(options): from allmydata.scripts import tahoe_mkdir diff --git a/src/allmydata/scripts/create_node.py b/src/allmydata/scripts/create_node.py index 2634e0915..9af9bf3ad 100644 --- a/src/allmydata/scripts/create_node.py +++ b/src/allmydata/scripts/create_node.py @@ -3,6 +3,11 @@ from __future__ import print_function import os import json +try: + from allmydata.scripts.types_ import SubCommands +except ImportError: + pass + from twisted.internet import reactor, defer from twisted.python.usage import UsageError from allmydata.scripts.common import BasedirOptions, NoDefaultBasedirOptions @@ -478,10 +483,10 @@ def create_introducer(config): subCommands = [ - ["create-node", None, CreateNodeOptions, "Create a node that acts as a client, server or both."], - ["create-client", None, CreateClientOptions, "Create a client node (with storage initially disabled)."], - ["create-introducer", None, CreateIntroducerOptions, "Create an introducer node."], -] + ("create-node", None, CreateNodeOptions, "Create a node that acts as a client, server or both."), + ("create-client", None, CreateClientOptions, "Create a client node (with storage initially disabled)."), + ("create-introducer", None, CreateIntroducerOptions, "Create an introducer node."), +] # type: SubCommands dispatch = { "create-node": create_node, diff --git a/src/allmydata/scripts/debug.py b/src/allmydata/scripts/debug.py index fd3f2b87c..be6b7e2bc 100644 --- a/src/allmydata/scripts/debug.py +++ b/src/allmydata/scripts/debug.py @@ -1,5 +1,10 @@ from __future__ import print_function +try: + from allmydata.scripts.types_ import SubCommands +except ImportError: + pass + # do not import any allmydata modules at this level. Do that from inside # individual functions instead. import struct, time, os, sys @@ -1051,8 +1056,8 @@ def do_debug(options): subCommands = [ - ["debug", None, DebugCommand, "debug subcommands: use 'tahoe debug' for a list."], - ] + ("debug", None, DebugCommand, "debug subcommands: use 'tahoe debug' for a list."), + ] # type: SubCommands dispatch = { "debug": do_debug, diff --git a/src/allmydata/scripts/runner.py b/src/allmydata/scripts/runner.py index 3436a1b84..999d7d353 100644 --- a/src/allmydata/scripts/runner.py +++ b/src/allmydata/scripts/runner.py @@ -4,6 +4,11 @@ import os, sys from six.moves import StringIO import six +try: + from allmydata.scripts.types_ import SubCommands +except ImportError: + pass + from twisted.python import usage from twisted.internet import defer, task, threads @@ -45,12 +50,12 @@ _control_node_dispatch = { } process_control_commands = [ - ["run", None, tahoe_run.RunOptions, "run a node without daemonizing"], - ["daemonize", None, tahoe_daemonize.DaemonizeOptions, "(deprecated) run a node in the background"], - ["start", None, tahoe_start.StartOptions, "(deprecated) start a node in the background and confirm it started"], - ["stop", None, tahoe_stop.StopOptions, "(deprecated) stop a node"], - ["restart", None, tahoe_restart.RestartOptions, "(deprecated) restart a node"], -] + ("run", None, tahoe_run.RunOptions, "run a node without daemonizing"), + ("daemonize", None, tahoe_daemonize.DaemonizeOptions, "(deprecated) run a node in the background"), + ("start", None, tahoe_start.StartOptions, "(deprecated) start a node in the background and confirm it started"), + ("stop", None, tahoe_stop.StopOptions, "(deprecated) stop a node"), + ("restart", None, tahoe_restart.RestartOptions, "(deprecated) restart a node"), +] # type: SubCommands class Options(usage.Options): diff --git a/src/allmydata/scripts/stats_gatherer.py b/src/allmydata/scripts/stats_gatherer.py index 26848a23c..b16ce689e 100644 --- a/src/allmydata/scripts/stats_gatherer.py +++ b/src/allmydata/scripts/stats_gatherer.py @@ -2,6 +2,11 @@ from __future__ import print_function import os +try: + from allmydata.scripts.types_ import SubCommands +except ImportError: + pass + # Python 2 compatibility from future.utils import PY2 if PY2: @@ -93,8 +98,8 @@ def create_stats_gatherer(config): return 0 subCommands = [ - ["create-stats-gatherer", None, CreateStatsGathererOptions, "Create a stats-gatherer service."], -] + ("create-stats-gatherer", None, CreateStatsGathererOptions, "Create a stats-gatherer service."), +] # type: SubCommands dispatch = { "create-stats-gatherer": create_stats_gatherer, diff --git a/src/allmydata/scripts/tahoe_invite.py b/src/allmydata/scripts/tahoe_invite.py index cca4216e3..f2d978f55 100644 --- a/src/allmydata/scripts/tahoe_invite.py +++ b/src/allmydata/scripts/tahoe_invite.py @@ -3,6 +3,11 @@ from __future__ import print_function import json from os.path import join +try: + from allmydata.scripts.types_ import SubCommands +except ImportError: + pass + from twisted.python import usage from twisted.internet import defer, reactor @@ -104,7 +109,7 @@ def invite(options): subCommands = [ ("invite", None, InviteOptions, "Invite a new node to this grid"), -] +] # type: SubCommands dispatch = { "invite": invite, diff --git a/src/allmydata/scripts/types_.py b/src/allmydata/scripts/types_.py new file mode 100644 index 000000000..58f88722b --- /dev/null +++ b/src/allmydata/scripts/types_.py @@ -0,0 +1,10 @@ +from typing import List, Tuple, Type +from allmydata.scripts.common import BaseOptions + + +# Historically, subcommands were implemented as lists, but due to a +# [designed contraint in mypy](https://stackoverflow.com/a/52559625/70170), +# a Tuple is required. +SubCommand = Tuple[str, None, Type[BaseOptions], str] + +SubCommands = List[SubCommand] From 572d7b2e02b2ba54c8c4db1592bac1898b48aba1 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 29 Nov 2020 15:52:04 -0500 Subject: [PATCH 38/97] Ignore error when untyped Module has no dispatch. --- src/allmydata/scripts/runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/scripts/runner.py b/src/allmydata/scripts/runner.py index 999d7d353..e56130d87 100644 --- a/src/allmydata/scripts/runner.py +++ b/src/allmydata/scripts/runner.py @@ -113,7 +113,7 @@ class Options(usage.Options): create_dispatch = {} for module in (create_node, stats_gatherer): - create_dispatch.update(module.dispatch) + create_dispatch.update(module.dispatch) # type: ignore def parse_options(argv, config=None): if not config: From b1b3a2341517d2b993e21f6764797aa3f6c01d4f Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 29 Nov 2020 15:56:39 -0500 Subject: [PATCH 39/97] Fix type errors with CPUUsageMonitor subclasses with float POLL_INTERVAL. --- src/allmydata/stats.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/stats.py b/src/allmydata/stats.py index f669b0861..6de323b73 100644 --- a/src/allmydata/stats.py +++ b/src/allmydata/stats.py @@ -78,7 +78,7 @@ class LoadMonitor(service.MultiService): @implementer(IStatsProducer) class CPUUsageMonitor(service.MultiService): HISTORY_LENGTH = 15 - POLL_INTERVAL = 60 + POLL_INTERVAL = 60 # type: float def __init__(self): service.MultiService.__init__(self) From 1768377aecc4d7e563e0b42d0c79d8d0f2d7d8a5 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 29 Nov 2020 16:04:56 -0500 Subject: [PATCH 40/97] Ignore error in DummyStorage --- src/allmydata/test/storage_plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/storage_plugin.py b/src/allmydata/test/storage_plugin.py index 4a1f84531..24081ae09 100644 --- a/src/allmydata/test/storage_plugin.py +++ b/src/allmydata/test/storage_plugin.py @@ -48,7 +48,7 @@ class RIDummy(RemoteInterface): -@implementer(IFoolscapStoragePlugin) +@implementer(IFoolscapStoragePlugin) # type: ignore # todo: make stubs for twisted @attr.s class DummyStorage(object): name = attr.ib() From 3653d7ed16c6b5d365664aca504af0c388e99752 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 29 Nov 2020 16:05:09 -0500 Subject: [PATCH 41/97] Ignore type checks on Referenceable objects. Ref warner/foolscap#78. --- src/allmydata/test/storage_plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/storage_plugin.py b/src/allmydata/test/storage_plugin.py index 24081ae09..b3464a88f 100644 --- a/src/allmydata/test/storage_plugin.py +++ b/src/allmydata/test/storage_plugin.py @@ -107,7 +107,7 @@ class GetCounter(Resource, object): @implementer(RIDummy) @attr.s(frozen=True) -class DummyStorageServer(object): +class DummyStorageServer(object): # type: ignore # warner/foolscap#78 get_anonymous_storage_server = attr.ib() def remote_just_some_method(self): From 2514196b275ad65b9e546b4c54898a43be949edd Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 29 Nov 2020 16:06:46 -0500 Subject: [PATCH 42/97] Suppress typing error in DummyStorageClient --- src/allmydata/test/storage_plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/storage_plugin.py b/src/allmydata/test/storage_plugin.py index b3464a88f..ba11776e6 100644 --- a/src/allmydata/test/storage_plugin.py +++ b/src/allmydata/test/storage_plugin.py @@ -116,7 +116,7 @@ class DummyStorageServer(object): # type: ignore # warner/foolscap#78 @implementer(IStorageServer) @attr.s -class DummyStorageClient(object): +class DummyStorageClient(object): # type: ignore # incomplete implementation get_rref = attr.ib() configuration = attr.ib() announcement = attr.ib() From ffa19d1c07e6a876fcebe8e8edbe968b10c842a9 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 29 Nov 2020 16:08:22 -0500 Subject: [PATCH 43/97] Suppress typing errors in common Nodes --- src/allmydata/test/common.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/common.py b/src/allmydata/test/common.py index a420dd3ba..a68a56ba5 100644 --- a/src/allmydata/test/common.py +++ b/src/allmydata/test/common.py @@ -391,7 +391,7 @@ class DummyProducer(object): pass @implementer(IImmutableFileNode) -class FakeCHKFileNode(object): +class FakeCHKFileNode(object): # type: ignore # incomplete implementation """I provide IImmutableFileNode, but all of my data is stored in a class-level dictionary.""" @@ -529,7 +529,7 @@ def create_chk_filenode(contents, all_contents): @implementer(IMutableFileNode, ICheckable) -class FakeMutableFileNode(object): +class FakeMutableFileNode(object): # type: ignore # incomplete implementation """I provide IMutableFileNode, but all of my data is stored in a class-level dictionary.""" From 53ff740f0ec2a13b50673fb989190cc73bee60f6 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 29 Nov 2020 16:14:10 -0500 Subject: [PATCH 44/97] Suppress type check error on NativeStorageServerWithVersion --- src/allmydata/test/test_storage_client.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/allmydata/test/test_storage_client.py b/src/allmydata/test/test_storage_client.py index fa3a34b15..f2be9ad1e 100644 --- a/src/allmydata/test/test_storage_client.py +++ b/src/allmydata/test/test_storage_client.py @@ -90,7 +90,12 @@ from allmydata.interfaces import ( SOME_FURL = b"pb://abcde@nowhere/fake" -class NativeStorageServerWithVersion(NativeStorageServer): + +# type checks fail with: +# Cannot determine consistent method resolution order (MRO) for "NativeStorageServerWithVersion" +# even though class hierarchy is single-inheritance. Probably `implementer` +# wrappers are affecting the MRO. +class NativeStorageServerWithVersion(NativeStorageServer): # type: ignore def __init__(self, version): # note: these instances won't work for anything other than # get_available_space() because we don't upcall From 7507e84a18902f5874a9aaaabf45dee92ec05510 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 29 Nov 2020 16:19:24 -0500 Subject: [PATCH 45/97] Suppress errors in no_network --- src/allmydata/test/no_network.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/allmydata/test/no_network.py b/src/allmydata/test/no_network.py index 59ab807bb..1e8c519e1 100644 --- a/src/allmydata/test/no_network.py +++ b/src/allmydata/test/no_network.py @@ -67,7 +67,7 @@ class Marker(object): fireNow = partial(defer.succeed, None) -@implementer(IRemoteReference) +@implementer(IRemoteReference) # type: ignore # todo: write stubs for foolscap class LocalWrapper(object): """ A ``LocalWrapper`` presents the remote reference interface to a local @@ -212,9 +212,12 @@ class NoNetworkServer(object): return _StorageServer(lambda: self.rref) def get_version(self): return self.rref.version + def start_connecting(self, trigger_cb): + raise NotImplementedError + @implementer(IStorageBroker) -class NoNetworkStorageBroker(object): +class NoNetworkStorageBroker(object): # type: ignore # missing many methods def get_servers_for_psi(self, peer_selection_index): def _permuted(server): seed = server.get_permutation_seed() @@ -258,7 +261,7 @@ def create_no_network_client(basedir): return defer.succeed(client) -class _NoNetworkClient(_Client): +class _NoNetworkClient(_Client): # type: ignore # Cannot determine consistent MRO order """ Overrides all _Client networking functionality to do nothing. """ From 13cd780231bf3185104d930d748c75b9d324eabc Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 29 Nov 2020 16:22:12 -0500 Subject: [PATCH 46/97] Prefer sys.maxsize to sys.maxint. --- src/allmydata/test/check_memory.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/allmydata/test/check_memory.py b/src/allmydata/test/check_memory.py index 41cf6e1d7..4f50b383b 100644 --- a/src/allmydata/test/check_memory.py +++ b/src/allmydata/test/check_memory.py @@ -499,13 +499,13 @@ if __name__ == '__main__': mode = "upload" if len(sys.argv) > 1: mode = sys.argv[1] - if sys.maxint == 2147483647: + if sys.maxsize == 2147483647: bits = "32" - elif sys.maxint == 9223372036854775807: + elif sys.maxsize == 9223372036854775807: bits = "64" else: bits = "?" - print("%s-bit system (sys.maxint=%d)" % (bits, sys.maxint)) + print("%s-bit system (sys.maxsize=%d)" % (bits, sys.maxsize)) # put the logfile and stats.out in _test_memory/ . These stick around. # put the nodes and other files in _test_memory/test/ . These are # removed each time we run. From b0803a2ac054234f341ec1650695ce988d926f25 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 29 Nov 2020 16:24:20 -0500 Subject: [PATCH 47/97] Suppress errors in test_web due to ambiguous MRO --- src/allmydata/test/web/test_web.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/web/test_web.py b/src/allmydata/test/web/test_web.py index 326569a26..a5fbe5a51 100644 --- a/src/allmydata/test/web/test_web.py +++ b/src/allmydata/test/web/test_web.py @@ -189,7 +189,7 @@ class FakeHistory(object): def list_all_helper_statuses(self): return [] -class FakeDisplayableServer(StubServer): +class FakeDisplayableServer(StubServer): # type: ignore # Cannot determine MRO def __init__(self, serverid, nickname, connected, last_connect_time, last_loss_time, last_rx_time): StubServer.__init__(self, serverid) @@ -255,7 +255,7 @@ class FakeStorageServer(service.MultiService): def on_status_changed(self, cb): cb(self) -class FakeClient(_Client): +class FakeClient(_Client): # type: ignore # Cannot determine MRO def __init__(self): # don't upcall to Client.__init__, since we only want to initialize a # minimal subset From fc19d1baf4a503dabc1a69b93adfd3200b5e413b Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 29 Nov 2020 16:26:36 -0500 Subject: [PATCH 48/97] Suppress errors in test_sftp --- src/allmydata/test/test_sftp.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/allmydata/test/test_sftp.py b/src/allmydata/test/test_sftp.py index b6f1fbc8a..1ff0363e8 100644 --- a/src/allmydata/test/test_sftp.py +++ b/src/allmydata/test/test_sftp.py @@ -9,18 +9,15 @@ from twisted.python.failure import Failure from twisted.internet.error import ProcessDone, ProcessTerminated from allmydata.util import deferredutil -conch_interfaces = None -sftp = None -sftpd = None - try: from twisted.conch import interfaces as conch_interfaces from twisted.conch.ssh import filetransfer as sftp from allmydata.frontends import sftpd except ImportError as e: + conch_interfaces = sftp = sftpd = None # type: ignore conch_unavailable_reason = e else: - conch_unavailable_reason = None + conch_unavailable_reason = None # type: ignore from allmydata.interfaces import IDirectoryNode, ExistingChildError, NoSuchChildError from allmydata.mutable.common import NotWriteableError From 54e45498367485d44ed6f886c50feb269a26c9e5 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 29 Nov 2020 16:30:13 -0500 Subject: [PATCH 49/97] Satisfy type check in test_helper.FakeClient. --- src/allmydata/test/test_helper.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/allmydata/test/test_helper.py b/src/allmydata/test/test_helper.py index 65c07135a..3faffbe0d 100644 --- a/src/allmydata/test/test_helper.py +++ b/src/allmydata/test/test_helper.py @@ -19,6 +19,12 @@ from functools import ( ) import attr +try: + from typing import List + from allmydata.introducer.client import IntroducerClient +except ImportError: + pass + from twisted.internet import defer from twisted.trial import unittest from twisted.application import service @@ -125,7 +131,7 @@ class FakeCHKCheckerAndUEBFetcher(object): )) class FakeClient(service.MultiService): - introducer_clients = [] + introducer_clients = [] # type: List[IntroducerClient] DEFAULT_ENCODING_PARAMETERS = {"k":25, "happy": 75, "n": 100, From 86f88a4aa5568876989f1bc222703cc9dd3d35fb Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 29 Nov 2020 16:32:19 -0500 Subject: [PATCH 50/97] Satisfy type checks in test_dirnode --- src/allmydata/test/test_dirnode.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/test_dirnode.py b/src/allmydata/test/test_dirnode.py index 48ffff45a..8af1567c9 100644 --- a/src/allmydata/test/test_dirnode.py +++ b/src/allmydata/test/test_dirnode.py @@ -1526,7 +1526,7 @@ class Packing(testutil.ReallyEqualMixin, unittest.TestCase): kids, fn.get_writekey(), deep_immutable=True) @implementer(IMutableFileNode) -class FakeMutableFile(object): +class FakeMutableFile(object): # type: ignore # incomplete interface counter = 0 def __init__(self, initial_contents=""): data = self._get_initial_contents(initial_contents) @@ -1587,7 +1587,7 @@ class FakeNodeMaker(NodeMaker): def create_mutable_file(self, contents="", keysize=None, version=None): return defer.succeed(FakeMutableFile(contents)) -class FakeClient2(_Client): +class FakeClient2(_Client): # type: ignore # ambiguous MRO def __init__(self): self.nodemaker = FakeNodeMaker(None, None, None, None, None, From d2d3f1f4a9a6aecb482859c96e5732c41847051f Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 29 Nov 2020 16:33:25 -0500 Subject: [PATCH 51/97] Suppress type errors in test_checker --- src/allmydata/test/test_checker.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/allmydata/test/test_checker.py b/src/allmydata/test/test_checker.py index 85b894b1f..23ce44762 100644 --- a/src/allmydata/test/test_checker.py +++ b/src/allmydata/test/test_checker.py @@ -62,7 +62,7 @@ class FakeClient(object): @implementer(IServer) -class FakeServer(object): +class FakeServer(object): # type: ignore # incomplete interface def get_name(self): return "fake name" @@ -75,7 +75,7 @@ class FakeServer(object): @implementer(ICheckResults) -class FakeCheckResults(object): +class FakeCheckResults(object): # type: ignore # incomplete interface def __init__(self, si=None, healthy=False, recoverable=False, @@ -106,7 +106,7 @@ class FakeCheckResults(object): @implementer(ICheckAndRepairResults) -class FakeCheckAndRepairResults(object): +class FakeCheckAndRepairResults(object): # type: ignore # incomplete interface def __init__(self, si=None, repair_attempted=False, From d2e2a22f6230f880c133f05cad73bd9b8490a909 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 4 Dec 2020 10:34:21 -0500 Subject: [PATCH 52/97] Run typechecks in CircleCI --- .circleci/config.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index afa3fafa1..ff14d6dd3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -91,6 +91,9 @@ workflows: - "build-porting-depgraph": <<: *DOCKERHUB_CONTEXT + - "typechecks": + <<: *DOCKERHUB_CONTEXT + images: # Build the Docker images used by the ci jobs. This makes the ci jobs # faster and takes various spurious failures out of the critical path. @@ -475,6 +478,17 @@ jobs: . /tmp/venv/bin/activate ./misc/python3/depgraph.sh + typechecks: + docker: + - <<: *DOCKERHUB_AUTH + image: "jaraco/multipy-tox" + + steps: + - "checkout" + - run: + name: "Validate Types" + command: tox -e typechecks + build-image: &BUILD_IMAGE # This is a template for a job to build a Docker image that has as much of # the setup as we can manage already done and baked in. This cuts down on From 3eb975748a9a6c32c7972ce4c498ec5afabfda9d Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 4 Dec 2020 19:56:51 -0500 Subject: [PATCH 53/97] Ignore type checks in allmydata. --- src/allmydata/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/__init__.py b/src/allmydata/__init__.py index 15d5fb240..a43781158 100644 --- a/src/allmydata/__init__.py +++ b/src/allmydata/__init__.py @@ -14,7 +14,7 @@ __all__ = [ __version__ = "unknown" try: - from allmydata._version import __version__ + from allmydata._version import __version__ # type: ignore except ImportError: # We're running in a tree that hasn't run update_version, and didn't # come with a _version.py, so we don't know what our version is. @@ -24,7 +24,7 @@ except ImportError: full_version = "unknown" branch = "unknown" try: - from allmydata._version import full_version, branch + from allmydata._version import full_version, branch # type: ignore except ImportError: # We're running in a tree that hasn't run update_version, and didn't # come with a _version.py, so we don't know what our full version or From 950ca189326703daf8bb84188507ca7324efc16e Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 18 Dec 2020 09:23:27 -0500 Subject: [PATCH 54/97] Ignores no longer needed. --- src/allmydata/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/__init__.py b/src/allmydata/__init__.py index a43781158..15d5fb240 100644 --- a/src/allmydata/__init__.py +++ b/src/allmydata/__init__.py @@ -14,7 +14,7 @@ __all__ = [ __version__ = "unknown" try: - from allmydata._version import __version__ # type: ignore + from allmydata._version import __version__ except ImportError: # We're running in a tree that hasn't run update_version, and didn't # come with a _version.py, so we don't know what our version is. @@ -24,7 +24,7 @@ except ImportError: full_version = "unknown" branch = "unknown" try: - from allmydata._version import full_version, branch # type: ignore + from allmydata._version import full_version, branch except ImportError: # We're running in a tree that hasn't run update_version, and didn't # come with a _version.py, so we don't know what our full version or From 99da74fffb1b013602ae847dfd4756ccb5e61c3d Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 18 Dec 2020 09:58:56 -0500 Subject: [PATCH 55/97] Change comment to clarify that it's the implementation that's incomplete. --- src/allmydata/test/test_checker.py | 6 +++--- src/allmydata/test/test_dirnode.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/allmydata/test/test_checker.py b/src/allmydata/test/test_checker.py index 23ce44762..d24ad4ef4 100644 --- a/src/allmydata/test/test_checker.py +++ b/src/allmydata/test/test_checker.py @@ -62,7 +62,7 @@ class FakeClient(object): @implementer(IServer) -class FakeServer(object): # type: ignore # incomplete interface +class FakeServer(object): # type: ignore # incomplete implementation def get_name(self): return "fake name" @@ -75,7 +75,7 @@ class FakeServer(object): # type: ignore # incomplete interface @implementer(ICheckResults) -class FakeCheckResults(object): # type: ignore # incomplete interface +class FakeCheckResults(object): # type: ignore # incomplete implementation def __init__(self, si=None, healthy=False, recoverable=False, @@ -106,7 +106,7 @@ class FakeCheckResults(object): # type: ignore # incomplete interface @implementer(ICheckAndRepairResults) -class FakeCheckAndRepairResults(object): # type: ignore # incomplete interface +class FakeCheckAndRepairResults(object): # type: ignore # incomplete implementation def __init__(self, si=None, repair_attempted=False, diff --git a/src/allmydata/test/test_dirnode.py b/src/allmydata/test/test_dirnode.py index 8af1567c9..68fcdd54b 100644 --- a/src/allmydata/test/test_dirnode.py +++ b/src/allmydata/test/test_dirnode.py @@ -1526,7 +1526,7 @@ class Packing(testutil.ReallyEqualMixin, unittest.TestCase): kids, fn.get_writekey(), deep_immutable=True) @implementer(IMutableFileNode) -class FakeMutableFile(object): # type: ignore # incomplete interface +class FakeMutableFile(object): # type: ignore # incomplete implementation counter = 0 def __init__(self, initial_contents=""): data = self._get_initial_contents(initial_contents) From 51b0b201b49733280702ed901baa06e23d75af85 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 18 Dec 2020 10:33:30 -0500 Subject: [PATCH 56/97] Expand comment to provide more context. --- src/allmydata/web/private.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/allmydata/web/private.py b/src/allmydata/web/private.py index a86c869c4..405ca75e7 100644 --- a/src/allmydata/web/private.py +++ b/src/allmydata/web/private.py @@ -61,7 +61,12 @@ class IToken(ICredentials): pass -# Shoobx/mypy-zope#26 +# Workaround for Shoobx/mypy-zope#26, where without suitable +# stubs for twisted classes (ICredentials), IToken does not +# appear to be an Interface. The proper fix appears to be to +# create stubs for twisted +# (https://twistedmatrix.com/trac/ticket/9717). For now, +# bypassing the inline decorator syntax works around the issue. _itoken_impl = implementer(IToken) From efd0aef2847814c1c43a14db3ccc1cde01a95119 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 18 Dec 2020 10:36:43 -0500 Subject: [PATCH 57/97] Indicate that unichr is Python 2 only. --- src/allmydata/windows/fixups.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/windows/fixups.py b/src/allmydata/windows/fixups.py index c5ba3bb57..e98aa8a67 100644 --- a/src/allmydata/windows/fixups.py +++ b/src/allmydata/windows/fixups.py @@ -219,7 +219,7 @@ def initialize(): def unmangle(s): return re.sub( u'\\x7F[0-9a-fA-F]*\\;', - # type ignored for 'unichr' + # type ignored for 'unichr' (Python 2 only) lambda m: unichr(int(m.group(0)[1:-1], 16)), # type: ignore s, ) From ea0c10ef83d98f019472c744ad81f3c5a8f5fab1 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 18 Dec 2020 10:51:56 -0500 Subject: [PATCH 58/97] Remove set_size, unused --- src/allmydata/immutable/encode.py | 3 --- src/allmydata/interfaces.py | 5 ----- 2 files changed, 8 deletions(-) diff --git a/src/allmydata/immutable/encode.py b/src/allmydata/immutable/encode.py index e743ce766..9351df501 100644 --- a/src/allmydata/immutable/encode.py +++ b/src/allmydata/immutable/encode.py @@ -711,6 +711,3 @@ class Encoder(object): return self.uri_extension_data def get_uri_extension_hash(self): return self.uri_extension_hash - - def set_size(self, size): - raise NotImplementedError() diff --git a/src/allmydata/interfaces.py b/src/allmydata/interfaces.py index 95b1fdf63..edd16ea08 100644 --- a/src/allmydata/interfaces.py +++ b/src/allmydata/interfaces.py @@ -1825,11 +1825,6 @@ class IEncoder(Interface): willing to receive data. """ - def set_size(size): - """Specify the number of bytes that will be encoded. This must be - peformed before get_serialized_params() can be called. - """ - def set_encrypted_uploadable(u): """Provide a source of encrypted upload data. 'u' must implement IEncryptedUploadable. From d051791e9530e5e64d49942cdfd6afc0a9a98c17 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 18 Dec 2020 10:53:24 -0500 Subject: [PATCH 59/97] Add reference to ticket. --- src/allmydata/immutable/offloaded.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/allmydata/immutable/offloaded.py b/src/allmydata/immutable/offloaded.py index 53a1f911a..2d2c5c1f5 100644 --- a/src/allmydata/immutable/offloaded.py +++ b/src/allmydata/immutable/offloaded.py @@ -499,6 +499,7 @@ class LocalCiphertextReader(AskUntilSuccessMixin): # ??. I'm not sure if it makes sense to forward the close message. return self.call("close") + # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3561 def set_upload_status(self, upload_status): raise NotImplementedError From 090031cbfc33510e7eebc627f313c94c548ea91e Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 18 Dec 2020 10:55:20 -0500 Subject: [PATCH 60/97] Remove confirm_share_allocation from interface (unused). --- src/allmydata/immutable/upload.py | 3 --- src/allmydata/interfaces.py | 6 ------ 2 files changed, 9 deletions(-) diff --git a/src/allmydata/immutable/upload.py b/src/allmydata/immutable/upload.py index 171b71eff..e6da4812a 100644 --- a/src/allmydata/immutable/upload.py +++ b/src/allmydata/immutable/upload.py @@ -388,9 +388,6 @@ class PeerSelector(object): def add_peers(self, peerids=None): raise NotImplementedError - def confirm_share_allocation(self, peerid, shnum): - raise NotImplementedError - class _QueryStatistics(object): diff --git a/src/allmydata/interfaces.py b/src/allmydata/interfaces.py index edd16ea08..e460854f8 100644 --- a/src/allmydata/interfaces.py +++ b/src/allmydata/interfaces.py @@ -860,12 +860,6 @@ class IPeerSelector(Interface): peer selection begins. """ - def confirm_share_allocation(peerid, shnum): - """ - Confirm that an allocated peer=>share pairing has been - successfully established. - """ - def add_peers(peerids=set): """ Update my internal state to include the peers in peerids as From 0e248cb4ef03f40da7e29c1bcd8a1c390ea00105 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 18 Dec 2020 10:59:53 -0500 Subject: [PATCH 61/97] Declare signing key as required in introducer client publish. --- src/allmydata/introducer/client.py | 2 +- src/allmydata/introducer/interfaces.py | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/allmydata/introducer/client.py b/src/allmydata/introducer/client.py index 62642d0af..0e0a56442 100644 --- a/src/allmydata/introducer/client.py +++ b/src/allmydata/introducer/client.py @@ -198,7 +198,7 @@ class IntroducerClient(service.Service, Referenceable): ann_d.update(ann) return ann_d - def publish(self, service_name, ann, signing_key=None): + def publish(self, service_name, ann, signing_key): # we increment the seqnum every time we publish something new current_seqnum, current_nonce = self._sequencer() diff --git a/src/allmydata/introducer/interfaces.py b/src/allmydata/introducer/interfaces.py index 9f08f1943..24fd3945f 100644 --- a/src/allmydata/introducer/interfaces.py +++ b/src/allmydata/introducer/interfaces.py @@ -73,7 +73,7 @@ class IIntroducerClient(Interface): publish their services to the rest of the world, and I help them learn about services available on other nodes.""" - def publish(service_name, ann, signing_key=None): + def publish(service_name, ann, signing_key): """Publish the given announcement dictionary (which must be JSON-serializable), plus some additional keys, to the world. @@ -83,8 +83,7 @@ class IIntroducerClient(Interface): the signing_key, if present, otherwise it is derived from the 'anonymous-storage-FURL' key. - If signing_key= is set to an instance of SigningKey, it will be - used to sign the announcement.""" + signing_key (a SigningKey) will be used to sign the announcement.""" def subscribe_to(service_name, callback, *args, **kwargs): """Call this if you will eventually want to use services with the From c2d2aba83f025824e23f2051700ad2fd54063799 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 18 Dec 2020 11:05:55 -0500 Subject: [PATCH 62/97] Add reference to ticket. --- src/allmydata/mutable/filenode.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/allmydata/mutable/filenode.py b/src/allmydata/mutable/filenode.py index 4613a918b..39e8b76be 100644 --- a/src/allmydata/mutable/filenode.py +++ b/src/allmydata/mutable/filenode.py @@ -1206,5 +1206,6 @@ class MutableFileVersion(object): mode=mode) return u.update() + # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3562 def get_servermap(self): raise NotImplementedError From 189608e11388988b512aa00f225911ff61afc80c Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 18 Dec 2020 11:07:30 -0500 Subject: [PATCH 63/97] Remove GENERATED_FILES, unused --- src/allmydata/node.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/allmydata/node.py b/src/allmydata/node.py index 7622d5bc3..6d0b2da04 100644 --- a/src/allmydata/node.py +++ b/src/allmydata/node.py @@ -713,7 +713,6 @@ class Node(service.MultiService): """ NODETYPE = "unknown NODETYPE" CERTFILE = "node.pem" - GENERATED_FILES = [] # type: ignore def __init__(self, config, main_tub, control_tub, i2p_provider, tor_provider): """ From 602a06e5cba818cb927c45566a610eb3e0d6eadc Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 18 Dec 2020 11:14:07 -0500 Subject: [PATCH 64/97] Extract Parameters type in scripts.types_. --- src/allmydata/scripts/cli.py | 5 ++--- src/allmydata/scripts/common.py | 7 ++++--- src/allmydata/scripts/types_.py | 4 +++- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/src/allmydata/scripts/cli.py b/src/allmydata/scripts/cli.py index 2bb91472f..e4cd8aa22 100644 --- a/src/allmydata/scripts/cli.py +++ b/src/allmydata/scripts/cli.py @@ -3,8 +3,7 @@ from __future__ import print_function import os.path, re, fnmatch try: - from typing import List, Sequence, Any - from allmydata.scripts.types_ import SubCommands + from allmydata.scripts.types_ import SubCommands, Parameters except ImportError: pass @@ -26,7 +25,7 @@ class FileStoreOptions(BaseOptions): "This overrides the URL found in the --node-directory ."], ["dir-cap", None, None, "Specify which dirnode URI should be used as the 'tahoe' alias."] - ] # type: List[Sequence[Any]] + ] # type: Parameters def postOptions(self): self["quiet"] = self.parent["quiet"] diff --git a/src/allmydata/scripts/common.py b/src/allmydata/scripts/common.py index b501c7e6a..42c26bb90 100644 --- a/src/allmydata/scripts/common.py +++ b/src/allmydata/scripts/common.py @@ -5,7 +5,8 @@ import codecs from os.path import join try: - from typing import Optional, Sequence, List, Any + from typing import Optional + from .types_ import Parameters except ImportError: pass @@ -69,7 +70,7 @@ class BasedirOptions(BaseOptions): optParameters = [ ["basedir", "C", None, "Specify which Tahoe base directory should be used. [default: %s]" % quote_local_unicode_path(_default_nodedir)], - ] # type: List[Sequence[Any]] + ] # type: Parameters def parseArgs(self, basedir=None): # This finds the node-directory option correctly even if we are in a subcommand. @@ -106,7 +107,7 @@ class NoDefaultBasedirOptions(BasedirOptions): optParameters = [ ["basedir", "C", None, "Specify which Tahoe base directory should be used."], - ] # type: List[Sequence[Any]] + ] # type: Parameters # This is overridden in order to ensure we get a "Wrong number of arguments." # error when more than one argument is given. diff --git a/src/allmydata/scripts/types_.py b/src/allmydata/scripts/types_.py index 58f88722b..3937cb803 100644 --- a/src/allmydata/scripts/types_.py +++ b/src/allmydata/scripts/types_.py @@ -1,4 +1,4 @@ -from typing import List, Tuple, Type +from typing import List, Tuple, Type, Sequence, Any from allmydata.scripts.common import BaseOptions @@ -8,3 +8,5 @@ from allmydata.scripts.common import BaseOptions SubCommand = Tuple[str, None, Type[BaseOptions], str] SubCommands = List[SubCommand] + +Parameters = List[Sequence[Any]] From 6b6b8f8378f1e3da826a50d9dcbe95e63ec8faa1 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 26 Dec 2020 11:17:35 -0500 Subject: [PATCH 65/97] Push IURI implementers down to the classes that actually implement it. --- src/allmydata/uri.py | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/src/allmydata/uri.py b/src/allmydata/uri.py index d7f4782cd..ce7794dda 100644 --- a/src/allmydata/uri.py +++ b/src/allmydata/uri.py @@ -494,7 +494,6 @@ class MDMFVerifierURI(_BaseURI): return self -@implementer(IURI, IDirnodeURI) class _DirectoryBaseURI(_BaseURI): def __init__(self, filenode_uri=None): self._filenode_uri = filenode_uri @@ -540,14 +539,8 @@ class _DirectoryBaseURI(_BaseURI): def get_storage_index(self): return self._filenode_uri.get_storage_index() - def get_readonly(self): - raise NotImplementedError() - def is_readonly(self): - raise NotImplementedError() - - -@implementer(IDirectoryURI) +@implementer(IURI, IDirectoryURI) class DirectoryURI(_DirectoryBaseURI): BASE_STRING=b'URI:DIR2:' @@ -566,7 +559,7 @@ class DirectoryURI(_DirectoryBaseURI): return ReadonlyDirectoryURI(self._filenode_uri.get_readonly()) -@implementer(IReadonlyDirectoryURI) +@implementer(IURI, IReadonlyDirectoryURI) class ReadonlyDirectoryURI(_DirectoryBaseURI): BASE_STRING=b'URI:DIR2-RO:' @@ -585,6 +578,7 @@ class ReadonlyDirectoryURI(_DirectoryBaseURI): return self +@implementer(IURI, IDirnodeURI) class _ImmutableDirectoryBaseURI(_DirectoryBaseURI): def __init__(self, filenode_uri=None): if filenode_uri: @@ -622,7 +616,7 @@ class LiteralDirectoryURI(_ImmutableDirectoryBaseURI): return None -@implementer(IDirectoryURI) +@implementer(IURI, IDirectoryURI) class MDMFDirectoryURI(_DirectoryBaseURI): BASE_STRING=b'URI:DIR2-MDMF:' @@ -644,7 +638,7 @@ class MDMFDirectoryURI(_DirectoryBaseURI): return MDMFDirectoryURIVerifier(self._filenode_uri.get_verify_cap()) -@implementer(IReadonlyDirectoryURI) +@implementer(IURI, IReadonlyDirectoryURI) class ReadonlyMDMFDirectoryURI(_DirectoryBaseURI): BASE_STRING=b'URI:DIR2-MDMF-RO:' @@ -682,7 +676,7 @@ def wrap_dirnode_cap(filecap): raise AssertionError("cannot interpret as a directory cap: %s" % filecap.__class__) -@implementer(IVerifierURI) +@implementer(IURI, IVerifierURI) class MDMFDirectoryURIVerifier(_DirectoryBaseURI): BASE_STRING=b'URI:DIR2-MDMF-Verifier:' @@ -707,7 +701,7 @@ class MDMFDirectoryURIVerifier(_DirectoryBaseURI): return self -@implementer(IVerifierURI) +@implementer(IURI, IVerifierURI) class DirectoryURIVerifier(_DirectoryBaseURI): BASE_STRING=b'URI:DIR2-Verifier:' From b65ef3cee6c6922c70edac842e28985ac7c20990 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 26 Dec 2020 11:22:39 -0500 Subject: [PATCH 66/97] Revert "Ignores no longer needed." This reverts commit 950ca189326703daf8bb84188507ca7324efc16e. --- src/allmydata/__init__.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/allmydata/__init__.py b/src/allmydata/__init__.py index 15d5fb240..3157c8c80 100644 --- a/src/allmydata/__init__.py +++ b/src/allmydata/__init__.py @@ -14,7 +14,9 @@ __all__ = [ __version__ = "unknown" try: - from allmydata._version import __version__ + # type ignored as it fails in CI + # (https://app.circleci.com/pipelines/github/tahoe-lafs/tahoe-lafs/1647/workflows/60ae95d4-abe8-492c-8a03-1ad3b9e42ed3/jobs/40972) + from allmydata._version import __version__ # type: ignore except ImportError: # We're running in a tree that hasn't run update_version, and didn't # come with a _version.py, so we don't know what our version is. @@ -24,7 +26,9 @@ except ImportError: full_version = "unknown" branch = "unknown" try: - from allmydata._version import full_version, branch + # type ignored as it fails in CI + # (https://app.circleci.com/pipelines/github/tahoe-lafs/tahoe-lafs/1647/workflows/60ae95d4-abe8-492c-8a03-1ad3b9e42ed3/jobs/40972) + from allmydata._version import full_version, branch # type: ignore except ImportError: # We're running in a tree that hasn't run update_version, and didn't # come with a _version.py, so we don't know what our full version or From ab2c544efcaad629f2d49173bfdc82e8de94d944 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 26 Dec 2020 11:50:06 -0500 Subject: [PATCH 67/97] Restore IDirnodeURI --- src/allmydata/uri.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/allmydata/uri.py b/src/allmydata/uri.py index ce7794dda..51671b0ac 100644 --- a/src/allmydata/uri.py +++ b/src/allmydata/uri.py @@ -494,6 +494,7 @@ class MDMFVerifierURI(_BaseURI): return self +@implementer(IDirnodeURI) class _DirectoryBaseURI(_BaseURI): def __init__(self, filenode_uri=None): self._filenode_uri = filenode_uri From dacdf7f12da89e214eec9b0543b15ef450b27ae3 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 26 Dec 2020 11:56:13 -0500 Subject: [PATCH 68/97] Add more detail and link to upstream issue for Twisted stubs. --- src/allmydata/test/storage_plugin.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/storage_plugin.py b/src/allmydata/test/storage_plugin.py index ba11776e6..17ec89078 100644 --- a/src/allmydata/test/storage_plugin.py +++ b/src/allmydata/test/storage_plugin.py @@ -47,8 +47,9 @@ class RIDummy(RemoteInterface): """ - -@implementer(IFoolscapStoragePlugin) # type: ignore # todo: make stubs for twisted +# type ignored due to missing stubs for Twisted +# https://twistedmatrix.com/trac/ticket/9717 +@implementer(IFoolscapStoragePlugin) # type: ignore @attr.s class DummyStorage(object): name = attr.ib() From 5396f9f97eaf5a541c2334eaf8ebdb5a2d3484f5 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 26 Dec 2020 12:02:47 -0500 Subject: [PATCH 69/97] Replace fixme with reference to foolscap issue. --- src/allmydata/storage/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py index b7df702d5..5f2ef3ac2 100644 --- a/src/allmydata/storage/server.py +++ b/src/allmydata/storage/server.py @@ -581,7 +581,7 @@ class StorageServer(service.MultiService, Referenceable): for share in six.viewvalues(shares): share.add_or_renew_lease(lease_info) - def slot_testv_and_readv_and_writev( # type: ignore # fixme + def slot_testv_and_readv_and_writev( # type: ignore # warner/foolscap#78 self, storage_index, secrets, From 1bf71fd69010a0a075f66076156f0e951042b8dd Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 26 Dec 2020 12:12:39 -0500 Subject: [PATCH 70/97] Replace todo with a ticket. --- src/allmydata/test/no_network.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/no_network.py b/src/allmydata/test/no_network.py index 1e8c519e1..ba497f81a 100644 --- a/src/allmydata/test/no_network.py +++ b/src/allmydata/test/no_network.py @@ -67,7 +67,7 @@ class Marker(object): fireNow = partial(defer.succeed, None) -@implementer(IRemoteReference) # type: ignore # todo: write stubs for foolscap +@implementer(IRemoteReference) # type: ignore # warner/foolscap#79 class LocalWrapper(object): """ A ``LocalWrapper`` presents the remote reference interface to a local From 01147f4627a10ecb4db5ddcdcb4a17f5d839a627 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 26 Dec 2020 12:33:22 -0500 Subject: [PATCH 71/97] Add reference to ticket for ambiguous MRO --- src/allmydata/test/no_network.py | 2 +- src/allmydata/test/test_dirnode.py | 2 +- src/allmydata/test/test_storage_client.py | 6 +----- src/allmydata/test/web/test_web.py | 4 ++-- 4 files changed, 5 insertions(+), 9 deletions(-) diff --git a/src/allmydata/test/no_network.py b/src/allmydata/test/no_network.py index ba497f81a..b9c9ecaeb 100644 --- a/src/allmydata/test/no_network.py +++ b/src/allmydata/test/no_network.py @@ -261,7 +261,7 @@ def create_no_network_client(basedir): return defer.succeed(client) -class _NoNetworkClient(_Client): # type: ignore # Cannot determine consistent MRO order +class _NoNetworkClient(_Client): # type: ignore # tahoe-lafs/ticket/3573 """ Overrides all _Client networking functionality to do nothing. """ diff --git a/src/allmydata/test/test_dirnode.py b/src/allmydata/test/test_dirnode.py index 68fcdd54b..6866bc88e 100644 --- a/src/allmydata/test/test_dirnode.py +++ b/src/allmydata/test/test_dirnode.py @@ -1587,7 +1587,7 @@ class FakeNodeMaker(NodeMaker): def create_mutable_file(self, contents="", keysize=None, version=None): return defer.succeed(FakeMutableFile(contents)) -class FakeClient2(_Client): # type: ignore # ambiguous MRO +class FakeClient2(_Client): # type: ignore # tahoe-lafs/ticket/3573 def __init__(self): self.nodemaker = FakeNodeMaker(None, None, None, None, None, diff --git a/src/allmydata/test/test_storage_client.py b/src/allmydata/test/test_storage_client.py index f2be9ad1e..bcbfffa1e 100644 --- a/src/allmydata/test/test_storage_client.py +++ b/src/allmydata/test/test_storage_client.py @@ -91,11 +91,7 @@ from allmydata.interfaces import ( SOME_FURL = b"pb://abcde@nowhere/fake" -# type checks fail with: -# Cannot determine consistent method resolution order (MRO) for "NativeStorageServerWithVersion" -# even though class hierarchy is single-inheritance. Probably `implementer` -# wrappers are affecting the MRO. -class NativeStorageServerWithVersion(NativeStorageServer): # type: ignore +class NativeStorageServerWithVersion(NativeStorageServer): # type: ignore # tahoe-lafs/ticket/3573 def __init__(self, version): # note: these instances won't work for anything other than # get_available_space() because we don't upcall diff --git a/src/allmydata/test/web/test_web.py b/src/allmydata/test/web/test_web.py index a5fbe5a51..ef0446077 100644 --- a/src/allmydata/test/web/test_web.py +++ b/src/allmydata/test/web/test_web.py @@ -189,7 +189,7 @@ class FakeHistory(object): def list_all_helper_statuses(self): return [] -class FakeDisplayableServer(StubServer): # type: ignore # Cannot determine MRO +class FakeDisplayableServer(StubServer): # type: ignore # tahoe-lafs/ticket/3573 def __init__(self, serverid, nickname, connected, last_connect_time, last_loss_time, last_rx_time): StubServer.__init__(self, serverid) @@ -255,7 +255,7 @@ class FakeStorageServer(service.MultiService): def on_status_changed(self, cb): cb(self) -class FakeClient(_Client): # type: ignore # Cannot determine MRO +class FakeClient(_Client): # type: ignore # tahoe-lafs/ticket/3573 def __init__(self): # don't upcall to Client.__init__, since we only want to initialize a # minimal subset From 04ab4dec3be3fdd07f7a85bfe50c3133cef30910 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 26 Dec 2020 13:01:27 -0500 Subject: [PATCH 72/97] Extract function and annotate it to satisfy typechecks for _Config._basedir --- src/allmydata/node.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/allmydata/node.py b/src/allmydata/node.py index 89d5bcb90..fabb255f6 100644 --- a/src/allmydata/node.py +++ b/src/allmydata/node.py @@ -23,6 +23,11 @@ from base64 import b32decode, b32encode from errno import ENOENT, EPERM from warnings import warn +try: + from typing import Union +except ImportError: + pass + import attr # On Python 2 this will be the backported package. @@ -273,6 +278,11 @@ def _error_about_old_config_files(basedir, generated_files): raise e +def ensure_text_and_abspath_expanduser_unicode(basedir): + # type: (Union[bytes, str]) -> str + return abspath_expanduser_unicode(ensure_text(basedir)) + + @attr.s class _Config(object): """ @@ -300,8 +310,8 @@ class _Config(object): config = attr.ib(validator=attr.validators.instance_of(configparser.ConfigParser)) portnum_fname = attr.ib() _basedir = attr.ib( - converter=lambda basedir: abspath_expanduser_unicode(ensure_text(basedir)), - ) + converter=ensure_text_and_abspath_expanduser_unicode, + ) # type: str config_path = attr.ib( validator=attr.validators.optional( attr.validators.instance_of(FilePath), From 3fd46f94009101fdcfff8617ccb335bab255c57f Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 26 Dec 2020 13:02:16 -0500 Subject: [PATCH 73/97] Ignore additional attribute on the function. --- src/allmydata/test/test_node.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/test_node.py b/src/allmydata/test/test_node.py index 1e0f3020c..e44fd5743 100644 --- a/src/allmydata/test/test_node.py +++ b/src/allmydata/test/test_node.py @@ -564,7 +564,7 @@ class TestMissingPorts(unittest.TestCase): config = config_from_string(self.basedir, "portnum", config_data) with self.assertRaises(PortAssignmentRequired): _tub_portlocation(config, None, None) - test_listen_on_zero_with_host.todo = native_str( + test_listen_on_zero_with_host.todo = native_str( # type: ignore "https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3563" ) From cc5a1046d9d936712577717f1ba0409378d279e6 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 26 Dec 2020 13:07:12 -0500 Subject: [PATCH 74/97] Define type for IntroducerService.VERSION, accepting bytes or str as keys for now. --- src/allmydata/introducer/server.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/allmydata/introducer/server.py b/src/allmydata/introducer/server.py index 237c30315..339c5a0ac 100644 --- a/src/allmydata/introducer/server.py +++ b/src/allmydata/introducer/server.py @@ -15,6 +15,12 @@ from past.builtins import long from six import ensure_text import time, os.path, textwrap + +try: + from typing import Any, Dict, Union +except ImportError: + pass + from zope.interface import implementer from twisted.application import service from twisted.internet import defer @@ -147,10 +153,12 @@ class IntroducerService(service.MultiService, Referenceable): name = "introducer" # v1 is the original protocol, added in 1.0 (but only advertised starting # in 1.3), removed in 1.12. v2 is the new signed protocol, added in 1.10 - VERSION = { #"http://allmydata.org/tahoe/protocols/introducer/v1": { }, + # TODO: reconcile bytes/str for keys + VERSION = { + #"http://allmydata.org/tahoe/protocols/introducer/v1": { }, b"http://allmydata.org/tahoe/protocols/introducer/v2": { }, b"application-version": allmydata.__full_version__.encode("utf-8"), - } + } # type: Dict[Union[bytes, str], Any] def __init__(self): service.MultiService.__init__(self) From 854c22e1ca44baa943e36c964c9d06cfa71f8809 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 26 Dec 2020 13:09:57 -0500 Subject: [PATCH 75/97] Use compatible import for urllib.parse.quote. --- src/allmydata/test/web/test_root.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/allmydata/test/web/test_root.py b/src/allmydata/test/web/test_root.py index 0715c8102..2ea418047 100644 --- a/src/allmydata/test/web/test_root.py +++ b/src/allmydata/test/web/test_root.py @@ -1,8 +1,7 @@ import time -from urllib import ( - quote, -) + +from six.moves.urllib.parse import quote from bs4 import ( BeautifulSoup, From 652222116665f6876e476e5a03cf1129fbc1c789 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 26 Dec 2020 13:12:11 -0500 Subject: [PATCH 76/97] Suppress error on SpyHandler interface. --- src/allmydata/test/test_storage_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/test_storage_client.py b/src/allmydata/test/test_storage_client.py index 45a99979f..8500d6bff 100644 --- a/src/allmydata/test/test_storage_client.py +++ b/src/allmydata/test/test_storage_client.py @@ -570,7 +570,7 @@ class SpyEndpoint(object): return d -@implementer(IConnectionHintHandler) +@implementer(IConnectionHintHandler) # type: ignore # warner/foolscap#78 @attr.s class SpyHandler(object): """ From 9780f8bfdcb29120149e4a49254a20fd7576e4ce Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 26 Dec 2020 13:23:28 -0500 Subject: [PATCH 77/97] Add newsfragment --- newsfragments/3399.feature | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/3399.feature diff --git a/newsfragments/3399.feature b/newsfragments/3399.feature new file mode 100644 index 000000000..d30a91679 --- /dev/null +++ b/newsfragments/3399.feature @@ -0,0 +1 @@ +Added 'typechecks' environment for tox running mypy and performing static typechecks. From 4a9d3bde5b53bb7ce9ae38ca03f416f9a4f5aad4 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 26 Dec 2020 13:30:32 -0500 Subject: [PATCH 78/97] Exclude allmydata.scripts.types_ module from PythonTwoRegressions. --- src/allmydata/test/test_python2_regressions.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/allmydata/test/test_python2_regressions.py b/src/allmydata/test/test_python2_regressions.py index 84484f1cf..5c6a654c1 100644 --- a/src/allmydata/test/test_python2_regressions.py +++ b/src/allmydata/test/test_python2_regressions.py @@ -16,6 +16,7 @@ from testtools.matchers import ( BLACKLIST = { "allmydata.test.check_load", "allmydata.windows.registry", + "allmydata.scripts.types_", } From 3951257cd7090188d33f3efabeca263c954912d0 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 27 Dec 2020 16:17:59 -0500 Subject: [PATCH 79/97] Switch to mypy-zope main branch. --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 26e056dc1..706899ebe 100644 --- a/tox.ini +++ b/tox.ini @@ -117,7 +117,7 @@ commands = skip_install = True deps = mypy - git+https://github.com/jaraco/mypy-zope@bugfix/21-InterfaceClass-subclass + git+https://github.com/Shoobx/mypy-zope git+https://github.com/jaraco/foolscap@bugfix/75-use-metaclass commands = mypy src From 116c59142d46a033cf4d9ac8615a43fee78cf022 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 12 Jan 2021 11:26:43 -0500 Subject: [PATCH 80/97] Port to Python 3. --- src/allmydata/test/web/test_logs.py | 4 ++++ src/allmydata/util/_python3.py | 1 + 2 files changed, 5 insertions(+) diff --git a/src/allmydata/test/web/test_logs.py b/src/allmydata/test/web/test_logs.py index 4895ed6f0..ca8e5b918 100644 --- a/src/allmydata/test/web/test_logs.py +++ b/src/allmydata/test/web/test_logs.py @@ -9,6 +9,10 @@ from __future__ import ( division, ) +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + from testtools.matchers import ( Equals, ) diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 14db70735..eca06058b 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -186,6 +186,7 @@ PORTED_TEST_MODULES = [ "allmydata.test.test_util", "allmydata.test.web.test_common", "allmydata.test.web.test_grid", + "allmydata.test.web.test_logs", "allmydata.test.web.test_status", "allmydata.test.web.test_util", "allmydata.test.web.test_webish", From d99c94753c1e222e0e5cb9eca79369f5f955f123 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 12 Jan 2021 11:38:37 -0500 Subject: [PATCH 81/97] On Python 3 we need to make sure bytes get written to the websocket. --- src/allmydata/web/logs.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/allmydata/web/logs.py b/src/allmydata/web/logs.py index 0ba8b17e9..896dce418 100644 --- a/src/allmydata/web/logs.py +++ b/src/allmydata/web/logs.py @@ -5,6 +5,8 @@ from __future__ import ( division, ) +from future.builtins import str + import json from autobahn.twisted.resource import WebSocketResource @@ -49,7 +51,11 @@ class TokenAuthenticatedWebSocketServerProtocol(WebSocketServerProtocol): """ # probably want a try/except around here? what do we do if # transmission fails or anything else bad happens? - self.sendMessage(json.dumps(message)) + encoded = json.dumps(message) + if isinstance(encoded, str): + # On Python 3 dumps() returns Unicode... + encoded = encoded.encode("utf-8") + self.sendMessage(encoded) def onOpen(self): """ From c2d69c53096fe1bd5894fd0cd198a475a2d592d5 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 12 Jan 2021 11:41:19 -0500 Subject: [PATCH 82/97] Merge all log tests into one test module. --- src/allmydata/test/test_websocket_logs.py | 54 ---------------------- src/allmydata/test/web/test_logs.py | 56 +++++++++++++++++++++++ 2 files changed, 56 insertions(+), 54 deletions(-) delete mode 100644 src/allmydata/test/test_websocket_logs.py diff --git a/src/allmydata/test/test_websocket_logs.py b/src/allmydata/test/test_websocket_logs.py deleted file mode 100644 index e666a4902..000000000 --- a/src/allmydata/test/test_websocket_logs.py +++ /dev/null @@ -1,54 +0,0 @@ -import json - -from twisted.trial import unittest -from twisted.internet.defer import inlineCallbacks - -from eliot import log_call - -from autobahn.twisted.testing import create_memory_agent, MemoryReactorClockResolver, create_pumper - -from allmydata.web.logs import TokenAuthenticatedWebSocketServerProtocol - - -class TestStreamingLogs(unittest.TestCase): - """ - Test websocket streaming of logs - """ - - def setUp(self): - self.reactor = MemoryReactorClockResolver() - self.pumper = create_pumper() - self.agent = create_memory_agent(self.reactor, self.pumper, TokenAuthenticatedWebSocketServerProtocol) - return self.pumper.start() - - def tearDown(self): - return self.pumper.stop() - - @inlineCallbacks - def test_one_log(self): - """ - write a single Eliot log and see it streamed via websocket - """ - - proto = yield self.agent.open( - transport_config=u"ws://localhost:1234/ws", - options={}, - ) - - messages = [] - def got_message(msg, is_binary=False): - messages.append(json.loads(msg)) - proto.on("message", got_message) - - @log_call(action_type=u"test:cli:some-exciting-action") - def do_a_thing(): - pass - - do_a_thing() - - proto.transport.loseConnection() - yield proto.is_closed - - self.assertEqual(len(messages), 2) - self.assertEqual("started", messages[0]["action_status"]) - self.assertEqual("succeeded", messages[1]["action_status"]) diff --git a/src/allmydata/test/web/test_logs.py b/src/allmydata/test/web/test_logs.py index ca8e5b918..5d697f910 100644 --- a/src/allmydata/test/web/test_logs.py +++ b/src/allmydata/test/web/test_logs.py @@ -1,5 +1,7 @@ """ Tests for ``allmydata.web.logs``. + +Ported to Python 3. """ from __future__ import ( @@ -13,6 +15,15 @@ from future.utils import PY2 if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 +import json + +from twisted.trial import unittest +from twisted.internet.defer import inlineCallbacks + +from eliot import log_call + +from autobahn.twisted.testing import create_memory_agent, MemoryReactorClockResolver, create_pumper + from testtools.matchers import ( Equals, ) @@ -41,6 +52,7 @@ from ..common import ( from ...web.logs import ( create_log_resources, + TokenAuthenticatedWebSocketServerProtocol, ) class StreamingEliotLogsTests(SyncTestCase): @@ -61,3 +73,47 @@ class StreamingEliotLogsTests(SyncTestCase): self.client.get(b"http:///v1"), succeeded(has_response_code(Equals(OK))), ) + + +class TestStreamingLogs(unittest.TestCase): + """ + Test websocket streaming of logs + """ + + def setUp(self): + self.reactor = MemoryReactorClockResolver() + self.pumper = create_pumper() + self.agent = create_memory_agent(self.reactor, self.pumper, TokenAuthenticatedWebSocketServerProtocol) + return self.pumper.start() + + def tearDown(self): + return self.pumper.stop() + + @inlineCallbacks + def test_one_log(self): + """ + write a single Eliot log and see it streamed via websocket + """ + + proto = yield self.agent.open( + transport_config=u"ws://localhost:1234/ws", + options={}, + ) + + messages = [] + def got_message(msg, is_binary=False): + messages.append(json.loads(msg)) + proto.on("message", got_message) + + @log_call(action_type=u"test:cli:some-exciting-action") + def do_a_thing(): + pass + + do_a_thing() + + proto.transport.loseConnection() + yield proto.is_closed + + self.assertEqual(len(messages), 2) + self.assertEqual("started", messages[0]["action_status"]) + self.assertEqual("succeeded", messages[1]["action_status"]) From 7e5e3291381ddf03625375e718658566da009e65 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 12 Jan 2021 11:44:27 -0500 Subject: [PATCH 83/97] Port to Python 3. --- src/allmydata/util/_python3.py | 1 + src/allmydata/web/logs.py | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index eca06058b..9917be84c 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -113,6 +113,7 @@ PORTED_MODULES = [ "allmydata.util.spans", "allmydata.util.statistics", "allmydata.util.time_format", + "allmydata.web.logs", "allmydata.webish", ] diff --git a/src/allmydata/web/logs.py b/src/allmydata/web/logs.py index 896dce418..6f15a3ca9 100644 --- a/src/allmydata/web/logs.py +++ b/src/allmydata/web/logs.py @@ -1,3 +1,6 @@ +""" +Ported to Python 3. +""" from __future__ import ( print_function, unicode_literals, @@ -5,8 +8,6 @@ from __future__ import ( division, ) -from future.builtins import str - import json from autobahn.twisted.resource import WebSocketResource From bd364feec5c9d178929c37179fdc9066f65d97f1 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 12 Jan 2021 13:22:43 -0500 Subject: [PATCH 84/97] Tests pass on Python 3. --- src/allmydata/test/web/test_introducer.py | 2 +- src/allmydata/web/introweb.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/allmydata/test/web/test_introducer.py b/src/allmydata/test/web/test_introducer.py index 929fba507..43f4d5934 100644 --- a/src/allmydata/test/web/test_introducer.py +++ b/src/allmydata/test/web/test_introducer.py @@ -213,7 +213,7 @@ class IntroducerRootTests(unittest.TestCase): resource = IntroducerRoot(introducer_node) response = json.loads( self.successResultOf( - render(resource, {"t": [b"json"]}), + render(resource, {b"t": [b"json"]}), ), ) self.assertEqual( diff --git a/src/allmydata/web/introweb.py b/src/allmydata/web/introweb.py index 6ec558e82..280d6cc26 100644 --- a/src/allmydata/web/introweb.py +++ b/src/allmydata/web/introweb.py @@ -26,10 +26,10 @@ class IntroducerRoot(MultiFormatResource): self.introducer_node = introducer_node self.introducer_service = introducer_node.getServiceNamed("introducer") # necessary as a root Resource - self.putChild("", self) + self.putChild(b"", self) static_dir = resource_filename("allmydata.web", "static") for filen in os.listdir(static_dir): - self.putChild(filen, static.File(os.path.join(static_dir, filen))) + self.putChild(filen.encode("utf-8"), static.File(os.path.join(static_dir, filen))) def _create_element(self): """ @@ -66,7 +66,7 @@ class IntroducerRoot(MultiFormatResource): announcement_summary[service_name] += 1 res[u"announcement_summary"] = announcement_summary - return json.dumps(res, indent=1) + b"\n" + return (json.dumps(res, indent=1) + "\n").encode("utf-8") class IntroducerRootElement(Element): From 8c41f60fdb2b6017f83497ed7a877f955fd7f673 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 12 Jan 2021 14:17:38 -0500 Subject: [PATCH 85/97] Port to Python 3. --- src/allmydata/test/web/test_introducer.py | 12 ++++++++++++ src/allmydata/util/_python3.py | 1 + 2 files changed, 13 insertions(+) diff --git a/src/allmydata/test/web/test_introducer.py b/src/allmydata/test/web/test_introducer.py index 43f4d5934..08d95bda9 100644 --- a/src/allmydata/test/web/test_introducer.py +++ b/src/allmydata/test/web/test_introducer.py @@ -1,3 +1,15 @@ +""" +Ported to Python 3. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + import json from os.path import join diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 9917be84c..63aa5bb0a 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -187,6 +187,7 @@ PORTED_TEST_MODULES = [ "allmydata.test.test_util", "allmydata.test.web.test_common", "allmydata.test.web.test_grid", + "allmydata.test.web.test_introducer", "allmydata.test.web.test_logs", "allmydata.test.web.test_status", "allmydata.test.web.test_util", From c076e1ee2646758a65c42100265e4ddb80e269ef Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 12 Jan 2021 14:24:11 -0500 Subject: [PATCH 86/97] Just fix all the putChild. --- src/allmydata/web/root.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/src/allmydata/web/root.py b/src/allmydata/web/root.py index fdc72ab71..f6316bff5 100644 --- a/src/allmydata/web/root.py +++ b/src/allmydata/web/root.py @@ -1,3 +1,5 @@ +from past.builtins import unicode + import os import time import urllib @@ -227,26 +229,25 @@ class Root(MultiFormatResource): self._client = client self._now_fn = now_fn - # Children need to be bytes; for now just doing these to make specific - # tests pass on Python 3, but eventually will do all them when this - # module is ported to Python 3 (if not earlier). self.putChild(b"uri", URIHandler(client)) - self.putChild("cap", URIHandler(client)) + self.putChild(b"cap", URIHandler(client)) # Handler for everything beneath "/private", an area of the resource # hierarchy which is only accessible with the private per-node API # auth token. - self.putChild("private", create_private_tree(client.get_auth_token)) + self.putChild(b"private", create_private_tree(client.get_auth_token)) - self.putChild("file", FileHandler(client)) - self.putChild("named", FileHandler(client)) - self.putChild("status", status.Status(client.get_history())) - self.putChild("statistics", status.Statistics(client.stats_provider)) + self.putChild(b"file", FileHandler(client)) + self.putChild(b"named", FileHandler(client)) + self.putChild(b"status", status.Status(client.get_history())) + self.putChild(b"statistics", status.Statistics(client.stats_provider)) static_dir = resource_filename("allmydata.web", "static") for filen in os.listdir(static_dir): + if isinstance(filen, unicode): + filen = filen.encode("utf-8") self.putChild(filen, static.File(os.path.join(static_dir, filen))) - self.putChild("report_incident", IncidentReporter()) + self.putChild(b"report_incident", IncidentReporter()) @exception_to_child def getChild(self, path, request): From 4940da47da1cd98ae9e0de3b5b80d11a8c2f573a Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 12 Jan 2021 14:24:17 -0500 Subject: [PATCH 87/97] Tests pass on Python 3. --- src/allmydata/test/web/test_private.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/web/test_private.py b/src/allmydata/test/web/test_private.py index 27ddbcf78..583f629f5 100644 --- a/src/allmydata/test/web/test_private.py +++ b/src/allmydata/test/web/test_private.py @@ -9,6 +9,8 @@ from __future__ import ( division, ) +from future.builtins import str + from testtools.matchers import ( Equals, ) @@ -56,6 +58,7 @@ class PrivacyTests(SyncTestCase): return super(PrivacyTests, self).setUp() def _authorization(self, scheme, value): + value = str(value, "utf-8") return Headers({ u"authorization": [u"{} {}".format(scheme, value)], }) @@ -90,7 +93,7 @@ class PrivacyTests(SyncTestCase): self.assertThat( self.client.head( b"http:///foo/bar", - headers=self._authorization(SCHEME, u"foo bar"), + headers=self._authorization(str(SCHEME, "utf-8"), b"foo bar"), ), succeeded(has_response_code(Equals(UNAUTHORIZED))), ) @@ -103,7 +106,7 @@ class PrivacyTests(SyncTestCase): self.assertThat( self.client.head( b"http:///foo/bar", - headers=self._authorization(SCHEME, self.token), + headers=self._authorization(str(SCHEME, "utf-8"), self.token), ), # It's a made up URL so we don't get a 200, either, but a 404. succeeded(has_response_code(Equals(NOT_FOUND))), From 03fb936716689447edb676c85044cabd79833f3d Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 12 Jan 2021 14:25:16 -0500 Subject: [PATCH 88/97] Port to Python 3. --- src/allmydata/test/web/test_private.py | 6 ++++++ src/allmydata/util/_python3.py | 1 + 2 files changed, 7 insertions(+) diff --git a/src/allmydata/test/web/test_private.py b/src/allmydata/test/web/test_private.py index 583f629f5..293796b1c 100644 --- a/src/allmydata/test/web/test_private.py +++ b/src/allmydata/test/web/test_private.py @@ -1,5 +1,7 @@ """ Tests for ``allmydata.web.private``. + +Ported to Python 3. """ from __future__ import ( @@ -9,6 +11,10 @@ from __future__ import ( division, ) +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + from future.builtins import str from testtools.matchers import ( diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 63aa5bb0a..7cd80335c 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -189,6 +189,7 @@ PORTED_TEST_MODULES = [ "allmydata.test.web.test_grid", "allmydata.test.web.test_introducer", "allmydata.test.web.test_logs", + "allmydata.test.web.test_private", "allmydata.test.web.test_status", "allmydata.test.web.test_util", "allmydata.test.web.test_webish", From 7a3e9ab43e520a531ab63178a06a1fe0160d6aed Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 12 Jan 2021 14:39:20 -0500 Subject: [PATCH 89/97] Tests pass on Python 3. --- src/allmydata/test/web/test_root.py | 8 ++++---- src/allmydata/web/root.py | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/allmydata/test/web/test_root.py b/src/allmydata/test/web/test_root.py index 0715c8102..497440d1c 100644 --- a/src/allmydata/test/web/test_root.py +++ b/src/allmydata/test/web/test_root.py @@ -1,6 +1,6 @@ import time -from urllib import ( +from urllib.parse import ( quote, ) @@ -77,7 +77,7 @@ class RenderSlashUri(unittest.TestCase): ) self.assertEqual( response_body, - "Invalid capability", + b"Invalid capability", ) @@ -92,7 +92,7 @@ class RenderServiceRow(unittest.TestCase): ann = {"anonymous-storage-FURL": "pb://w2hqnbaa25yw4qgcvghl5psa3srpfgw3@tcp:127.0.0.1:51309/vucto2z4fxment3vfxbqecblbf6zyp6x", "permutation-seed-base32": "w2hqnbaa25yw4qgcvghl5psa3srpfgw3", } - srv = NativeStorageServer("server_id", ann, None, {}, EMPTY_CLIENT_CONFIG) + srv = NativeStorageServer(b"server_id", ann, None, {}, EMPTY_CLIENT_CONFIG) srv.get_connection_status = lambda: ConnectionStatus(False, "summary", {}, 0, 0) class FakeClient(_Client): @@ -103,7 +103,7 @@ class RenderServiceRow(unittest.TestCase): tub_maker=None, node_config=EMPTY_CLIENT_CONFIG, ) - self.storage_broker.test_add_server("test-srv", srv) + self.storage_broker.test_add_server(b"test-srv", srv) root = RootElement(FakeClient(), time.time) req = DummyRequest(b"") diff --git a/src/allmydata/web/root.py b/src/allmydata/web/root.py index f6316bff5..5829da51e 100644 --- a/src/allmydata/web/root.py +++ b/src/allmydata/web/root.py @@ -2,7 +2,7 @@ from past.builtins import unicode import os import time -import urllib +from urllib.parse import quote as urlquote from hyperlink import DecodedURL, URL from pkg_resources import resource_filename @@ -83,7 +83,7 @@ class URIHandler(resource.Resource, object): # it seems Nevow was creating absolute URLs including # host/port whereas req.uri is absolute (but lacks host/port) redir_uri = URL.from_text(req.prePathURL().decode('utf8')) - redir_uri = redir_uri.child(urllib.quote(uri_arg).decode('utf8')) + redir_uri = redir_uri.child(urlquote(uri_arg)) # add back all the query args that AREN'T "?uri=" for k, values in req.args.items(): if k != b"uri": From 5d77282784b88dbcb3445d279b9c9d0afd6f6db2 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 12 Jan 2021 14:40:33 -0500 Subject: [PATCH 90/97] Ported to Python 3. --- src/allmydata/test/web/test_root.py | 12 ++++++++++++ src/allmydata/util/_python3.py | 1 + 2 files changed, 13 insertions(+) diff --git a/src/allmydata/test/web/test_root.py b/src/allmydata/test/web/test_root.py index 497440d1c..ca3cc695d 100644 --- a/src/allmydata/test/web/test_root.py +++ b/src/allmydata/test/web/test_root.py @@ -1,3 +1,15 @@ +""" +Ported to Python 3. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + import time from urllib.parse import ( diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 7cd80335c..575a69cb7 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -190,6 +190,7 @@ PORTED_TEST_MODULES = [ "allmydata.test.web.test_introducer", "allmydata.test.web.test_logs", "allmydata.test.web.test_private", + "allmydata.test.web.test_root", "allmydata.test.web.test_status", "allmydata.test.web.test_util", "allmydata.test.web.test_webish", From 6b0849490ad8cbbb23041b16e3850be6e837d30b Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 12 Jan 2021 14:40:46 -0500 Subject: [PATCH 91/97] News file. --- newsfragments/3589.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3589.minor diff --git a/newsfragments/3589.minor b/newsfragments/3589.minor new file mode 100644 index 000000000..e69de29bb From aace119790f92f691c586067f3554a842656c148 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 13 Jan 2021 09:55:54 -0500 Subject: [PATCH 92/97] Fix Python 3 issue with combining bytes and unicode. --- src/allmydata/web/root.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/allmydata/web/root.py b/src/allmydata/web/root.py index 5829da51e..e3d49cd66 100644 --- a/src/allmydata/web/root.py +++ b/src/allmydata/web/root.py @@ -243,9 +243,10 @@ class Root(MultiFormatResource): self.putChild(b"statistics", status.Statistics(client.stats_provider)) static_dir = resource_filename("allmydata.web", "static") for filen in os.listdir(static_dir): + child_path = filen if isinstance(filen, unicode): - filen = filen.encode("utf-8") - self.putChild(filen, static.File(os.path.join(static_dir, filen))) + child_path = filen.encode("utf-8") + self.putChild(child_path, static.File(os.path.join(static_dir, filen))) self.putChild(b"report_incident", IncidentReporter()) From c5669e16e0ba0943e83fd7f3e39a2569eb582743 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 13 Jan 2021 09:56:08 -0500 Subject: [PATCH 93/97] Fix flake. --- src/allmydata/test/web/test_private.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/allmydata/test/web/test_private.py b/src/allmydata/test/web/test_private.py index 293796b1c..b426b4d93 100644 --- a/src/allmydata/test/web/test_private.py +++ b/src/allmydata/test/web/test_private.py @@ -15,8 +15,6 @@ from future.utils import PY2 if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 -from future.builtins import str - from testtools.matchers import ( Equals, ) From a2dab7c89fed217603116b97fd7d5ca8d0426823 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 14 Jan 2021 09:40:10 -0500 Subject: [PATCH 94/97] Only do this on Python 3. --- src/allmydata/web/root.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/allmydata/web/root.py b/src/allmydata/web/root.py index e3d49cd66..b1ba501a9 100644 --- a/src/allmydata/web/root.py +++ b/src/allmydata/web/root.py @@ -1,3 +1,4 @@ +from future.utils import PY3 from past.builtins import unicode import os @@ -244,7 +245,7 @@ class Root(MultiFormatResource): static_dir = resource_filename("allmydata.web", "static") for filen in os.listdir(static_dir): child_path = filen - if isinstance(filen, unicode): + if PY3: child_path = filen.encode("utf-8") self.putChild(child_path, static.File(os.path.join(static_dir, filen))) From 42b31a28099ed7f5c732c4bfc6a71db46b303559 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 14 Jan 2021 15:58:18 -0500 Subject: [PATCH 95/97] Fix flake. --- src/allmydata/web/root.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/allmydata/web/root.py b/src/allmydata/web/root.py index b1ba501a9..0ef6b00d2 100644 --- a/src/allmydata/web/root.py +++ b/src/allmydata/web/root.py @@ -1,5 +1,4 @@ from future.utils import PY3 -from past.builtins import unicode import os import time From a01078ddec86e1d193c7940087d6506d376028d8 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 15 Jan 2021 15:00:57 -0500 Subject: [PATCH 96/97] Switch to one of our Docker images for typecheck CI --- .circleci/config.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ff14d6dd3..29b55ad5f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -481,13 +481,14 @@ jobs: typechecks: docker: - <<: *DOCKERHUB_AUTH - image: "jaraco/multipy-tox" + image: "tahoelafsci/ubuntu:18.04-py3" steps: - "checkout" - run: name: "Validate Types" - command: tox -e typechecks + command: | + /tmp/venv/bin/tox -e typechecks build-image: &BUILD_IMAGE # This is a template for a job to build a Docker image that has as much of From c7a4cdb44d801034de1e068f10aa3d8df8a4950f Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 15 Jan 2021 15:25:07 -0500 Subject: [PATCH 97/97] Rely on main branch of foolscap for typechecks. --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 706899ebe..2873bded8 100644 --- a/tox.ini +++ b/tox.ini @@ -118,7 +118,7 @@ skip_install = True deps = mypy git+https://github.com/Shoobx/mypy-zope - git+https://github.com/jaraco/foolscap@bugfix/75-use-metaclass + git+https://github.com/warner/foolscap commands = mypy src