From ee9d9d79848cfcc5e4e49e094db07c25082f9b64 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 11 Sep 2020 15:38:04 -0400 Subject: [PATCH 01/81] Add mypy checks as separate tox environment. --- mypy.ini | 2 ++ tox.ini | 7 +++++++ 2 files changed, 9 insertions(+) create mode 100644 mypy.ini diff --git a/mypy.ini b/mypy.ini new file mode 100644 index 000000000..976ba0294 --- /dev/null +++ b/mypy.ini @@ -0,0 +1,2 @@ +[mypy] +ignore_missing_imports = True diff --git a/tox.ini b/tox.ini index 597270e3a..f5c8f3f42 100644 --- a/tox.ini +++ b/tox.ini @@ -108,6 +108,13 @@ commands = # file. See pyproject.toml for legal values. python -m towncrier.check --pyproject towncrier.pyproject.toml + +[testenv:typechecks] +skip_install = True +deps = mypy +commands = mypy src + + [testenv:draftnews] passenv = TAHOE_LAFS_* PIP_* SUBUNITREPORTER_* USERPROFILE HOMEDRIVE HOMEPATH # see comment in [testenv] about "certifi" From ab54585558f32c92aa9e786df19c42d577e15a44 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 18 Sep 2020 15:49:21 -0400 Subject: [PATCH 02/81] Incorporate mypy-zope to support zope interfaces. --- mypy.ini | 1 + tox.ini | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/mypy.ini b/mypy.ini index 976ba0294..01cbb57a8 100644 --- a/mypy.ini +++ b/mypy.ini @@ -1,2 +1,3 @@ [mypy] ignore_missing_imports = True +plugins=mypy_zope:plugin diff --git a/tox.ini b/tox.ini index f5c8f3f42..1c232b4d7 100644 --- a/tox.ini +++ b/tox.ini @@ -111,7 +111,9 @@ commands = [testenv:typechecks] skip_install = True -deps = mypy +deps = + mypy + mypy-zope commands = mypy src From 6d2d82d7b7342be42d0392ccac0af3ed8882f41c Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Tue, 13 Oct 2020 21:06:07 -0400 Subject: [PATCH 03/81] Use pre-release versions of foolscap and mypy-zope with intended support for RemoteInterface subclasses. --- tox.ini | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 1c232b4d7..a13e840a4 100644 --- a/tox.ini +++ b/tox.ini @@ -113,7 +113,8 @@ commands = skip_install = True deps = mypy - mypy-zope + git+https://github.com/jaraco/mypy-zope@bugfix/21-InterfaceClass-subclass + git+https://github.com/jaraco/foolscap@bugfix/75-use-metaclass commands = mypy src From 4b559ffc3332222c99a86b0f75ea05a01a845321 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 31 Oct 2020 16:06:52 -0400 Subject: [PATCH 04/81] Add typechecks to tox run --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index a13e840a4..0cc0c4ac2 100644 --- a/tox.ini +++ b/tox.ini @@ -7,7 +7,7 @@ twisted = 1 [tox] -envlist = codechecks,py27,py36,pypy27 +envlist = typechecks,codechecks,py27,py36,pypy27 minversion = 2.4 [testenv] From f2ffa78198e756a2337a66463241127c38318509 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 20 Nov 2020 12:32:11 -0500 Subject: [PATCH 05/81] Define type of PollMixin._poll_should_ignore_these_errors --- src/allmydata/util/pollmixin.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/allmydata/util/pollmixin.py b/src/allmydata/util/pollmixin.py index 5d1716853..582bafe86 100644 --- a/src/allmydata/util/pollmixin.py +++ b/src/allmydata/util/pollmixin.py @@ -14,6 +14,12 @@ if PY2: from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import time + +try: + from typing import List +except ImportError: + pass + from twisted.internet import task class TimeoutError(Exception): @@ -23,7 +29,7 @@ class PollComplete(Exception): pass class PollMixin(object): - _poll_should_ignore_these_errors = [] + _poll_should_ignore_these_errors = [] # type: List[Exception] def poll(self, check_f, pollinterval=0.01, timeout=1000): # Return a Deferred, then call check_f periodically until it returns From ce3b775944fdd9070f57de1230c4b2f1c1b8a5b7 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 20 Nov 2020 12:33:41 -0500 Subject: [PATCH 06/81] Suppress typing error in test_python3 --- src/allmydata/test/test_python3.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/test_python3.py b/src/allmydata/test/test_python3.py index 80242f8a2..c1f0e83d6 100644 --- a/src/allmydata/test/test_python3.py +++ b/src/allmydata/test/test_python3.py @@ -44,7 +44,7 @@ class Python3PortingEffortTests(SynchronousTestCase): ), ), ) - test_finished_porting.todo = native_str( + test_finished_porting.todo = native_str( # type: ignore "https://tahoe-lafs.org/trac/tahoe-lafs/milestone/Support%20Python%203 should be completed", ) From d1ea36781a651ece31807c32f366ce6c596b8e70 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 20 Nov 2020 13:12:52 -0500 Subject: [PATCH 07/81] Add type declarations to check_load. --- src/allmydata/test/check_load.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/allmydata/test/check_load.py b/src/allmydata/test/check_load.py index 4058ddf77..21576ea3a 100644 --- a/src/allmydata/test/check_load.py +++ b/src/allmydata/test/check_load.py @@ -37,6 +37,11 @@ a mean of 10kB and a max of 100MB, so filesize=min(int(1.0/random(.0002)),1e8) import os, sys, httplib, binascii import urllib, json, random, time, urlparse +try: + from typing import Dict +except ImportError: + pass + # Python 2 compatibility from future.utils import PY2 if PY2: @@ -49,13 +54,13 @@ if sys.argv[1] == "--stats": DELAY = 10 MAXSAMPLES = 6 totals = [] - last_stats = {} + last_stats = {} # type: Dict[str, float] while True: - stats = {} + stats = {} # type: Dict[str, float] for sf in statsfiles: for line in open(sf, "r").readlines(): - name, value = line.split(":") - value = int(value.strip()) + name, str_value = line.split(":") + value = int(str_value.strip()) if name not in stats: stats[name] = 0 stats[name] += float(value) From 8da82e9ed55d363fb963b95e9b96fac79e6c1bb8 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 20 Nov 2020 13:47:23 -0500 Subject: [PATCH 08/81] Add workaround for Shoobx/mypy-zope#26. --- src/allmydata/web/private.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/allmydata/web/private.py b/src/allmydata/web/private.py index fea058405..a86c869c4 100644 --- a/src/allmydata/web/private.py +++ b/src/allmydata/web/private.py @@ -61,7 +61,11 @@ class IToken(ICredentials): pass -@implementer(IToken) +# Shoobx/mypy-zope#26 +_itoken_impl = implementer(IToken) + + +@_itoken_impl @attr.s class Token(object): proposed_token = attr.ib(type=bytes) From 25cce8b77ebbf6d195f938d39e0c00aa9e6262b9 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Mon, 23 Nov 2020 13:57:37 -0500 Subject: [PATCH 09/81] Suppress typing errors in fileutil, crawler, fixups. --- src/allmydata/storage/crawler.py | 2 +- src/allmydata/util/fileutil.py | 2 +- src/allmydata/windows/fixups.py | 7 ++++++- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/allmydata/storage/crawler.py b/src/allmydata/storage/crawler.py index 24042c38b..f13f7cb99 100644 --- a/src/allmydata/storage/crawler.py +++ b/src/allmydata/storage/crawler.py @@ -19,7 +19,7 @@ import os, time, struct try: import cPickle as pickle except ImportError: - import pickle + import pickle # type: ignore from twisted.internet import reactor from twisted.application import service from allmydata.storage.common import si_b2a diff --git a/src/allmydata/util/fileutil.py b/src/allmydata/util/fileutil.py index ea16c0d6a..e40e06180 100644 --- a/src/allmydata/util/fileutil.py +++ b/src/allmydata/util/fileutil.py @@ -311,7 +311,7 @@ def precondition_abspath(path): _getfullpathname = None try: - from nt import _getfullpathname + from nt import _getfullpathname # type: ignore except ImportError: pass diff --git a/src/allmydata/windows/fixups.py b/src/allmydata/windows/fixups.py index e7f045b95..c5ba3bb57 100644 --- a/src/allmydata/windows/fixups.py +++ b/src/allmydata/windows/fixups.py @@ -217,7 +217,12 @@ def initialize(): # Instead it "mangles" or escapes them using \x7F as an escape character, which we # unescape here. def unmangle(s): - return re.sub(u'\\x7F[0-9a-fA-F]*\\;', lambda m: unichr(int(m.group(0)[1:-1], 16)), s) + return re.sub( + u'\\x7F[0-9a-fA-F]*\\;', + # type ignored for 'unichr' + lambda m: unichr(int(m.group(0)[1:-1], 16)), # type: ignore + s, + ) try: argv = [unmangle(argv_unicode[i]).encode('utf-8') for i in xrange(0, argc.value)] From df31d7db5b23bbc1e12d83f44bf23c9cce316b25 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Mon, 23 Nov 2020 14:05:33 -0500 Subject: [PATCH 10/81] Suppress type error in Node.GENERATED_FILES, apparently unused. --- src/allmydata/node.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/node.py b/src/allmydata/node.py index 0dcd900aa..7622d5bc3 100644 --- a/src/allmydata/node.py +++ b/src/allmydata/node.py @@ -713,7 +713,7 @@ class Node(service.MultiService): """ NODETYPE = "unknown NODETYPE" CERTFILE = "node.pem" - GENERATED_FILES = [] + GENERATED_FILES = [] # type: ignore def __init__(self, config, main_tub, control_tub, i2p_provider, tor_provider): """ From dec6f6d64705b0f103dba3436598f42e96df365c Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Mon, 23 Nov 2020 14:08:23 -0500 Subject: [PATCH 11/81] Remove Interface subclass, as IURI is an interface. Fixes mypy error. --- src/allmydata/interfaces.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/interfaces.py b/src/allmydata/interfaces.py index 49dcf7646..537f6d655 100644 --- a/src/allmydata/interfaces.py +++ b/src/allmydata/interfaces.py @@ -682,7 +682,7 @@ class IURI(Interface): passing into init_from_string.""" -class IVerifierURI(Interface, IURI): +class IVerifierURI(IURI): def init_from_string(uri): """Accept a string (as created by my to_string() method) and populate this instance with its data. I am not normally called directly, From 5f40c562ebb0754c3eabfb0b1dddaeb6d9e4e3a6 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Mon, 23 Nov 2020 14:09:46 -0500 Subject: [PATCH 12/81] Remove self arguments to IProgress, which mypy caught as improper. --- src/allmydata/interfaces.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/interfaces.py b/src/allmydata/interfaces.py index 537f6d655..95b1fdf63 100644 --- a/src/allmydata/interfaces.py +++ b/src/allmydata/interfaces.py @@ -749,7 +749,7 @@ class IProgress(Interface): "Current amount of progress (in percentage)" ) - def set_progress(self, value): + def set_progress(value): """ Sets the current amount of progress. @@ -757,7 +757,7 @@ class IProgress(Interface): set_progress_total. """ - def set_progress_total(self, value): + def set_progress_total(value): """ Sets the total amount of expected progress From adf06889181c41c20db56aece8537a46945ae3ea Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Mon, 23 Nov 2020 14:15:39 -0500 Subject: [PATCH 13/81] Add a non-implementation of encode_proposal to satisfy interface. --- src/allmydata/codec.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/allmydata/codec.py b/src/allmydata/codec.py index a4baab4b6..19345959e 100644 --- a/src/allmydata/codec.py +++ b/src/allmydata/codec.py @@ -57,6 +57,10 @@ class CRSEncoder(object): return defer.succeed((shares, desired_share_ids)) + def encode_proposal(self, data, desired_share_ids=None): + raise NotImplementedError() + + @implementer(ICodecDecoder) class CRSDecoder(object): From 4998c4693fdfd900d0659000888e0e0a59346b8c Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 29 Nov 2020 13:46:52 -0500 Subject: [PATCH 14/81] Ignore type checks on Referenceable objects. Ref warner/foolscap#78. --- src/allmydata/immutable/offloaded.py | 4 ++-- src/allmydata/immutable/upload.py | 2 +- src/allmydata/storage/immutable.py | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/allmydata/immutable/offloaded.py b/src/allmydata/immutable/offloaded.py index d574b980d..4e18ad216 100644 --- a/src/allmydata/immutable/offloaded.py +++ b/src/allmydata/immutable/offloaded.py @@ -141,7 +141,7 @@ class CHKCheckerAndUEBFetcher(object): @implementer(interfaces.RICHKUploadHelper) -class CHKUploadHelper(Referenceable, upload.CHKUploader): +class CHKUploadHelper(Referenceable, upload.CHKUploader): # type: ignore # warner/foolscap#78 """I am the helper-server -side counterpart to AssistedUploader. I handle peer selection, encoding, and share pushing. I read ciphertext from the remote AssistedUploader. @@ -502,7 +502,7 @@ class LocalCiphertextReader(AskUntilSuccessMixin): @implementer(interfaces.RIHelper, interfaces.IStatsProducer) -class Helper(Referenceable): +class Helper(Referenceable): # type: ignore # warner/foolscap#78 """ :ivar dict[bytes, CHKUploadHelper] _active_uploads: For any uploads which have been started but not finished, a mapping from storage index to the diff --git a/src/allmydata/immutable/upload.py b/src/allmydata/immutable/upload.py index e77cbb30b..18f818504 100644 --- a/src/allmydata/immutable/upload.py +++ b/src/allmydata/immutable/upload.py @@ -1423,7 +1423,7 @@ class LiteralUploader(object): return self._status @implementer(RIEncryptedUploadable) -class RemoteEncryptedUploadable(Referenceable): +class RemoteEncryptedUploadable(Referenceable): # type: ignore # warner/foolscap#78 def __init__(self, encrypted_uploadable, upload_status): self._eu = IEncryptedUploadable(encrypted_uploadable) diff --git a/src/allmydata/storage/immutable.py b/src/allmydata/storage/immutable.py index 778c0ddf8..4b60d79f1 100644 --- a/src/allmydata/storage/immutable.py +++ b/src/allmydata/storage/immutable.py @@ -202,7 +202,7 @@ class ShareFile(object): @implementer(RIBucketWriter) -class BucketWriter(Referenceable): +class BucketWriter(Referenceable): # type: ignore # warner/foolscap#78 def __init__(self, ss, incominghome, finalhome, max_size, lease_info, canary): self.ss = ss @@ -301,7 +301,7 @@ class BucketWriter(Referenceable): @implementer(RIBucketReader) -class BucketReader(Referenceable): +class BucketReader(Referenceable): # type: ignore # warner/foolscap#78 def __init__(self, ss, sharefname, storage_index=None, shnum=None): self.ss = ss From 50f81aa25d7ad86c5c238bc2a1d70afce25de03f Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 29 Nov 2020 13:54:39 -0500 Subject: [PATCH 15/81] Update two methods of introducer.client.IntroducerClient to match the interface definition. --- src/allmydata/introducer/client.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/allmydata/introducer/client.py b/src/allmydata/introducer/client.py index 0a6352317..62642d0af 100644 --- a/src/allmydata/introducer/client.py +++ b/src/allmydata/introducer/client.py @@ -157,15 +157,15 @@ class IntroducerClient(service.Service, Referenceable): kwargs["facility"] = "tahoe.introducer.client" return log.msg(*args, **kwargs) - def subscribe_to(self, service_name, cb, *args, **kwargs): - self._local_subscribers.append( (service_name,cb,args,kwargs) ) + def subscribe_to(self, service_name, callback, *args, **kwargs): + self._local_subscribers.append( (service_name,callback,args,kwargs) ) self._subscribed_service_names.add(service_name) self._maybe_subscribe() for index,(ann,key_s,when) in self._inbound_announcements.items(): precondition(isinstance(key_s, str), key_s) servicename = index[0] if servicename == service_name: - eventually(cb, key_s, ann, *args, **kwargs) + eventually(callback, key_s, ann, *args, **kwargs) def _maybe_subscribe(self): if not self._publisher: @@ -198,7 +198,7 @@ class IntroducerClient(service.Service, Referenceable): ann_d.update(ann) return ann_d - def publish(self, service_name, ann, signing_key): + def publish(self, service_name, ann, signing_key=None): # we increment the seqnum every time we publish something new current_seqnum, current_nonce = self._sequencer() From bc3508ce6098d65ae0407047a2e02d3a1bacfee9 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 29 Nov 2020 14:01:05 -0500 Subject: [PATCH 16/81] Ignore type checks on cmp usage (awaiting Python 3 porting) --- src/allmydata/web/status.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/web/status.py b/src/allmydata/web/status.py index 7f6020a99..de15230fd 100644 --- a/src/allmydata/web/status.py +++ b/src/allmydata/web/status.py @@ -1335,7 +1335,7 @@ class Status(MultiFormatResource): active = [s for s in self._get_all_statuses() if s.get_active()] - active.sort(lambda a, b: cmp(a.get_started(), b.get_started())) + active.sort(lambda a, b: cmp(a.get_started(), b.get_started())) # type: ignore # py2 active.reverse() return active @@ -1343,7 +1343,7 @@ class Status(MultiFormatResource): recent = [s for s in self._get_all_statuses() if not s.get_active()] - recent.sort(lambda a, b: cmp(a.get_started(), b.get_started())) + recent.sort(lambda a, b: cmp(a.get_started(), b.get_started())) # type: ignore # py2 recent.reverse() return recent From 6ba7533168eff65454376987258748f99fb1057d Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 29 Nov 2020 14:04:59 -0500 Subject: [PATCH 17/81] Ignore failure on StorageServer.slot_testv_and_readv_and_writev, the implementation of which deviates from the interface spec substantially. --- src/allmydata/storage/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py index 8a8138f26..b7df702d5 100644 --- a/src/allmydata/storage/server.py +++ b/src/allmydata/storage/server.py @@ -581,7 +581,7 @@ class StorageServer(service.MultiService, Referenceable): for share in six.viewvalues(shares): share.add_or_renew_lease(lease_info) - def slot_testv_and_readv_and_writev( + def slot_testv_and_readv_and_writev( # type: ignore # fixme self, storage_index, secrets, From dca0840c350e5b188672e7410a4995b50a1dca60 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 29 Nov 2020 14:08:33 -0500 Subject: [PATCH 18/81] Add stubs for methods demanded by the interface --- src/allmydata/uri.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/allmydata/uri.py b/src/allmydata/uri.py index 2c367cafe..e95c86d96 100644 --- a/src/allmydata/uri.py +++ b/src/allmydata/uri.py @@ -535,6 +535,12 @@ class _DirectoryBaseURI(_BaseURI): def get_storage_index(self): return self._filenode_uri.get_storage_index() + def get_readonly(self): + raise NotImplementedError() + + def is_readonly(self): + raise NotImplementedError() + @implementer(IDirectoryURI) class DirectoryURI(_DirectoryBaseURI): From cc91b7c9edf0d5fc64525b410b50da0748f1180b Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 29 Nov 2020 14:15:24 -0500 Subject: [PATCH 19/81] Declare DirectoryURIVerifier type to allow subclass to override. --- src/allmydata/uri.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/allmydata/uri.py b/src/allmydata/uri.py index e95c86d96..d7f4782cd 100644 --- a/src/allmydata/uri.py +++ b/src/allmydata/uri.py @@ -22,6 +22,11 @@ from past.builtins import unicode, long import re +try: + from typing import Type +except ImportError: + pass + from zope.interface import implementer from twisted.python.components import registerAdapter @@ -707,7 +712,7 @@ class DirectoryURIVerifier(_DirectoryBaseURI): BASE_STRING=b'URI:DIR2-Verifier:' BASE_STRING_RE=re.compile(b'^'+BASE_STRING) - INNER_URI_CLASS=SSKVerifierURI + INNER_URI_CLASS=SSKVerifierURI # type: Type[IVerifierURI] def __init__(self, filenode_uri=None): if filenode_uri: From 7e757d2ec4e201d4e9a1e03be66bd5007eb5c094 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 29 Nov 2020 14:23:59 -0500 Subject: [PATCH 20/81] As _ImmutableFileNodeBase doesn't implement the interface, move the implementer declaration to LiteralFileNode --- src/allmydata/immutable/literal.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/allmydata/immutable/literal.py b/src/allmydata/immutable/literal.py index 68db478f3..6ed5571b9 100644 --- a/src/allmydata/immutable/literal.py +++ b/src/allmydata/immutable/literal.py @@ -19,7 +19,7 @@ from twisted.protocols import basic from allmydata.interfaces import IImmutableFileNode, ICheckable from allmydata.uri import LiteralFileURI -@implementer(IImmutableFileNode, ICheckable) + class _ImmutableFileNodeBase(object): def get_write_uri(self): @@ -56,6 +56,7 @@ class _ImmutableFileNodeBase(object): return not self == other +@implementer(IImmutableFileNode, ICheckable) class LiteralFileNode(_ImmutableFileNodeBase): def __init__(self, filecap): From e9ddcf5911dd75cceea8611dd7a134d7a7eb2fb1 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 29 Nov 2020 14:26:20 -0500 Subject: [PATCH 21/81] Implement set_size as required by the interface --- src/allmydata/immutable/encode.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/allmydata/immutable/encode.py b/src/allmydata/immutable/encode.py index 9351df501..e743ce766 100644 --- a/src/allmydata/immutable/encode.py +++ b/src/allmydata/immutable/encode.py @@ -711,3 +711,6 @@ class Encoder(object): return self.uri_extension_data def get_uri_extension_hash(self): return self.uri_extension_hash + + def set_size(self, size): + raise NotImplementedError() From 1248d65778242c81e914fa6096990c3e3e68fcba Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 29 Nov 2020 14:32:34 -0500 Subject: [PATCH 22/81] Declare types for BasedirOptions. Fixes several errors. --- src/allmydata/scripts/common.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/allmydata/scripts/common.py b/src/allmydata/scripts/common.py index 34266ee72..a633da655 100644 --- a/src/allmydata/scripts/common.py +++ b/src/allmydata/scripts/common.py @@ -4,6 +4,11 @@ import os, sys, urllib, textwrap import codecs from os.path import join +try: + from typing import Optional, Sequence, List +except ImportError: + pass + # Python 2 compatibility from future.utils import PY2 if PY2: @@ -64,7 +69,7 @@ class BasedirOptions(BaseOptions): optParameters = [ ["basedir", "C", None, "Specify which Tahoe base directory should be used. [default: %s]" % quote_local_unicode_path(_default_nodedir)], - ] + ] # type: List[Sequence[Optional[str]]] def parseArgs(self, basedir=None): # This finds the node-directory option correctly even if we are in a subcommand. From e0eb63929a56e46d622fa48b1be159b09cefc031 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 29 Nov 2020 14:35:17 -0500 Subject: [PATCH 23/81] Declare type for BaseOptions.description. Fixes many type errors. --- src/allmydata/scripts/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/scripts/common.py b/src/allmydata/scripts/common.py index a633da655..6d281e07c 100644 --- a/src/allmydata/scripts/common.py +++ b/src/allmydata/scripts/common.py @@ -45,7 +45,7 @@ class BaseOptions(usage.Options): def opt_version(self): raise usage.UsageError("--version not allowed on subcommands") - description = None + description = None # type: Optional[str] description_unwrapped = None def __str__(self): From 1b92da75fa01d94a8486e1a4108c3dcce10458e8 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 29 Nov 2020 14:44:17 -0500 Subject: [PATCH 24/81] Some subclasses use ints, so just go for Any --- src/allmydata/scripts/common.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/scripts/common.py b/src/allmydata/scripts/common.py index 6d281e07c..438b4a7e2 100644 --- a/src/allmydata/scripts/common.py +++ b/src/allmydata/scripts/common.py @@ -5,7 +5,7 @@ import codecs from os.path import join try: - from typing import Optional, Sequence, List + from typing import Optional, Sequence, List, Any except ImportError: pass @@ -69,7 +69,7 @@ class BasedirOptions(BaseOptions): optParameters = [ ["basedir", "C", None, "Specify which Tahoe base directory should be used. [default: %s]" % quote_local_unicode_path(_default_nodedir)], - ] # type: List[Sequence[Optional[str]]] + ] # type: List[Sequence[Any]] def parseArgs(self, basedir=None): # This finds the node-directory option correctly even if we are in a subcommand. From c3a22966e82fd698eab1d62b5416be53b863013b Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 29 Nov 2020 14:52:45 -0500 Subject: [PATCH 25/81] Add stubs for methods demanded by IPeerSelector --- src/allmydata/immutable/upload.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/allmydata/immutable/upload.py b/src/allmydata/immutable/upload.py index 18f818504..171b71eff 100644 --- a/src/allmydata/immutable/upload.py +++ b/src/allmydata/immutable/upload.py @@ -385,6 +385,12 @@ class PeerSelector(object): ) return self.happiness_mappings + def add_peers(self, peerids=None): + raise NotImplementedError + + def confirm_share_allocation(self, peerid, shnum): + raise NotImplementedError + class _QueryStatistics(object): From af172f6bff9b5de980473a02155840e50cdbae91 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 29 Nov 2020 14:55:13 -0500 Subject: [PATCH 26/81] Repeat type declaration from parent to avoid over-constraining this type for subclasses. --- src/allmydata/scripts/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/scripts/common.py b/src/allmydata/scripts/common.py index 438b4a7e2..81511681d 100644 --- a/src/allmydata/scripts/common.py +++ b/src/allmydata/scripts/common.py @@ -106,7 +106,7 @@ class NoDefaultBasedirOptions(BasedirOptions): optParameters = [ ["basedir", "C", None, "Specify which Tahoe base directory should be used."], - ] + ] # type: List[Sequence[Any]] # This is overridden in order to ensure we get a "Wrong number of arguments." # error when more than one argument is given. From 103bec6a1579373df3238c8b5ee09eb72f79718a Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 29 Nov 2020 14:58:28 -0500 Subject: [PATCH 27/81] On MutableFileNode, accept optional 'progress' parameter as declared by the interface. --- src/allmydata/mutable/filenode.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/mutable/filenode.py b/src/allmydata/mutable/filenode.py index 5afc84dec..54e9844d6 100644 --- a/src/allmydata/mutable/filenode.py +++ b/src/allmydata/mutable/filenode.py @@ -564,7 +564,7 @@ class MutableFileNode(object): return d - def upload(self, new_contents, servermap): + def upload(self, new_contents, servermap, progress=None): """ I overwrite the contents of the best recoverable version of this mutable file with new_contents, using servermap instead of From a75454a04f61c3e96b080553fe1f7e412b1d976b Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 29 Nov 2020 15:02:13 -0500 Subject: [PATCH 28/81] Add stub for MutableFileVersion.get_servermap --- src/allmydata/mutable/filenode.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/allmydata/mutable/filenode.py b/src/allmydata/mutable/filenode.py index 54e9844d6..e9cf23fa1 100644 --- a/src/allmydata/mutable/filenode.py +++ b/src/allmydata/mutable/filenode.py @@ -1205,3 +1205,6 @@ class MutableFileVersion(object): self._servermap, mode=mode) return u.update() + + def get_servermap(self): + raise NotImplementedError From 32b77c42394c337fd2bc38f6ff2be29e5acdd001 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 29 Nov 2020 15:06:12 -0500 Subject: [PATCH 29/81] Ignore interface violation in MutableFileVersion.download_to_data --- src/allmydata/mutable/filenode.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/mutable/filenode.py b/src/allmydata/mutable/filenode.py index e9cf23fa1..4613a918b 100644 --- a/src/allmydata/mutable/filenode.py +++ b/src/allmydata/mutable/filenode.py @@ -951,7 +951,7 @@ class MutableFileVersion(object): return self._servermap.size_of_version(self._version) - def download_to_data(self, fetch_privkey=False, progress=None): + def download_to_data(self, fetch_privkey=False, progress=None): # type: ignore # fixme """ I return a Deferred that fires with the contents of this readable object as a byte string. From 646297ddc3f95b9ab88b35d32a83782c8e2f178f Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 29 Nov 2020 15:08:46 -0500 Subject: [PATCH 30/81] Add stub for LocalCiphertextReader.set_upload_status --- src/allmydata/immutable/offloaded.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/allmydata/immutable/offloaded.py b/src/allmydata/immutable/offloaded.py index 4e18ad216..53a1f911a 100644 --- a/src/allmydata/immutable/offloaded.py +++ b/src/allmydata/immutable/offloaded.py @@ -499,6 +499,8 @@ class LocalCiphertextReader(AskUntilSuccessMixin): # ??. I'm not sure if it makes sense to forward the close message. return self.call("close") + def set_upload_status(self, upload_status): + raise NotImplementedError @implementer(interfaces.RIHelper, interfaces.IStatsProducer) From 67f0be8431157f965df50bd82e3098b9be3bbe12 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 29 Nov 2020 15:13:19 -0500 Subject: [PATCH 31/81] Prefer type(None) for better compatibility. --- src/allmydata/frontends/ftpd.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/allmydata/frontends/ftpd.py b/src/allmydata/frontends/ftpd.py index 0b18df85b..af5444969 100644 --- a/src/allmydata/frontends/ftpd.py +++ b/src/allmydata/frontends/ftpd.py @@ -1,7 +1,5 @@ from six import ensure_str -from types import NoneType - from zope.interface import implementer from twisted.application import service, strports from twisted.internet import defer @@ -317,7 +315,7 @@ class Dispatcher(object): class FTPServer(service.MultiService): def __init__(self, client, accountfile, accounturl, ftp_portstr): - precondition(isinstance(accountfile, (unicode, NoneType)), accountfile) + precondition(isinstance(accountfile, (unicode, type(None))), accountfile) service.MultiService.__init__(self) r = Dispatcher(client) From 8b991d3516166b62b7dfe4827ba7b1911de5f680 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 29 Nov 2020 15:16:45 -0500 Subject: [PATCH 32/81] Update DirectoryNode.set_uri to match interface spec. --- src/allmydata/dirnode.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/dirnode.py b/src/allmydata/dirnode.py index 59ebd73ba..6f052c3c7 100644 --- a/src/allmydata/dirnode.py +++ b/src/allmydata/dirnode.py @@ -554,7 +554,7 @@ class DirectoryNode(object): d = self.get_child_and_metadata(childnamex) return d - def set_uri(self, namex, writecap, readcap, metadata=None, overwrite=True): + def set_uri(self, namex, writecap, readcap=None, metadata=None, overwrite=True): precondition(isinstance(writecap, (str,type(None))), writecap) precondition(isinstance(readcap, (str,type(None))), readcap) From 6ea9003436e79163a865a0a30b6051f7811c1bd3 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 29 Nov 2020 15:19:11 -0500 Subject: [PATCH 33/81] Declare MultiFormatResource.formatDefault as optional string for subclass overrides. --- src/allmydata/web/common_py3.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/allmydata/web/common_py3.py b/src/allmydata/web/common_py3.py index 22f235790..080c24c19 100644 --- a/src/allmydata/web/common_py3.py +++ b/src/allmydata/web/common_py3.py @@ -4,6 +4,11 @@ Common utilities that are available from Python 3. Can eventually be merged back into allmydata.web.common. """ +try: + from typing import Optional +except ImportError: + pass + from twisted.web import resource, http from allmydata.util import abbreviate @@ -47,7 +52,7 @@ class MultiFormatResource(resource.Resource, object): format if nothing else is given as the ``formatDefault``. """ formatArgument = "t" - formatDefault = None + formatDefault = None # type: Optional[str] def render(self, req): """ From cb351607d81e21919c3c380c0ac55cd0dba3a5fc Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 29 Nov 2020 15:21:25 -0500 Subject: [PATCH 34/81] Repeat type declaration from parent to avoid over-constraining this type for subclasses. --- src/allmydata/scripts/cli.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/allmydata/scripts/cli.py b/src/allmydata/scripts/cli.py index 379e1d212..96dde65f8 100644 --- a/src/allmydata/scripts/cli.py +++ b/src/allmydata/scripts/cli.py @@ -1,6 +1,12 @@ from __future__ import print_function import os.path, re, fnmatch + +try: + from typing import List, Sequence, Any +except ImportError: + pass + from twisted.python import usage from allmydata.scripts.common import get_aliases, get_default_nodedir, \ DEFAULT_ALIAS, BaseOptions @@ -19,7 +25,7 @@ class FileStoreOptions(BaseOptions): "This overrides the URL found in the --node-directory ."], ["dir-cap", None, None, "Specify which dirnode URI should be used as the 'tahoe' alias."] - ] + ] # type: List[Sequence[Any]] def postOptions(self): self["quiet"] = self.parent["quiet"] From 6b772e7fdc14f473492a61e3e5abf507116077d0 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 29 Nov 2020 15:22:58 -0500 Subject: [PATCH 35/81] Declare type for BaseOptions.description_unwrapped. --- src/allmydata/scripts/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/scripts/common.py b/src/allmydata/scripts/common.py index 81511681d..b501c7e6a 100644 --- a/src/allmydata/scripts/common.py +++ b/src/allmydata/scripts/common.py @@ -46,7 +46,7 @@ class BaseOptions(usage.Options): raise usage.UsageError("--version not allowed on subcommands") description = None # type: Optional[str] - description_unwrapped = None + description_unwrapped = None # type: Optional[str] def __str__(self): width = int(os.environ.get('COLUMNS', '80')) From 41c341a3cc0e2a58847907cc9bee471de4d68519 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 29 Nov 2020 15:24:20 -0500 Subject: [PATCH 36/81] Prefer type(None) for better compatibility. --- src/allmydata/frontends/sftpd.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/allmydata/frontends/sftpd.py b/src/allmydata/frontends/sftpd.py index db914fa45..6ee42d505 100644 --- a/src/allmydata/frontends/sftpd.py +++ b/src/allmydata/frontends/sftpd.py @@ -1,6 +1,5 @@ import six import heapq, traceback, array, stat, struct -from types import NoneType from stat import S_IFREG, S_IFDIR from time import time, strftime, localtime @@ -267,7 +266,7 @@ def _attrs_to_metadata(attrs): def _direntry_for(filenode_or_parent, childname, filenode=None): - precondition(isinstance(childname, (unicode, NoneType)), childname=childname) + precondition(isinstance(childname, (unicode, type(None))), childname=childname) if childname is None: filenode_or_parent = filenode @@ -672,7 +671,7 @@ class GeneralSFTPFile(PrefixingLogMixin): self.log(".open(parent=%r, childname=%r, filenode=%r, metadata=%r)" % (parent, childname, filenode, metadata), level=OPERATIONAL) - precondition(isinstance(childname, (unicode, NoneType)), childname=childname) + precondition(isinstance(childname, (unicode, type(None))), childname=childname) precondition(filenode is None or IFileNode.providedBy(filenode), filenode=filenode) precondition(not self.closed, sftpfile=self) @@ -1194,7 +1193,7 @@ class SFTPUserHandler(ConchUser, PrefixingLogMixin): request = "._sync_heisenfiles(%r, %r, ignore=%r)" % (userpath, direntry, ignore) self.log(request, level=OPERATIONAL) - _assert(isinstance(userpath, str) and isinstance(direntry, (str, NoneType)), + _assert(isinstance(userpath, str) and isinstance(direntry, (str, type(None))), userpath=userpath, direntry=direntry) files = [] @@ -1219,7 +1218,7 @@ class SFTPUserHandler(ConchUser, PrefixingLogMixin): def _remove_heisenfile(self, userpath, parent, childname, file_to_remove): if noisy: self.log("._remove_heisenfile(%r, %r, %r, %r)" % (userpath, parent, childname, file_to_remove), level=NOISY) - _assert(isinstance(userpath, str) and isinstance(childname, (unicode, NoneType)), + _assert(isinstance(userpath, str) and isinstance(childname, (unicode, type(None))), userpath=userpath, childname=childname) direntry = _direntry_for(parent, childname) @@ -1246,7 +1245,7 @@ class SFTPUserHandler(ConchUser, PrefixingLogMixin): (existing_file, userpath, flags, _repr_flags(flags), parent, childname, filenode, metadata), level=NOISY) - _assert((isinstance(userpath, str) and isinstance(childname, (unicode, NoneType)) and + _assert((isinstance(userpath, str) and isinstance(childname, (unicode, type(None))) and (metadata is None or 'no-write' in metadata)), userpath=userpath, childname=childname, metadata=metadata) @@ -1977,7 +1976,7 @@ class Dispatcher(object): class SFTPServer(service.MultiService): def __init__(self, client, accountfile, accounturl, sftp_portstr, pubkey_file, privkey_file): - precondition(isinstance(accountfile, (unicode, NoneType)), accountfile) + precondition(isinstance(accountfile, (unicode, type(None))), accountfile) precondition(isinstance(pubkey_file, unicode), pubkey_file) precondition(isinstance(privkey_file, unicode), privkey_file) service.MultiService.__init__(self) From acbb6b3e93f0a1082e63db42fae20131908afe5e Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 29 Nov 2020 15:48:26 -0500 Subject: [PATCH 37/81] Convert subcommands to tuples instead of lists, as that's what mypy demands for heterogeneous sequences. --- src/allmydata/scripts/admin.py | 9 ++++-- src/allmydata/scripts/cli.py | 39 +++++++++++++------------ src/allmydata/scripts/create_node.py | 13 ++++++--- src/allmydata/scripts/debug.py | 9 ++++-- src/allmydata/scripts/runner.py | 17 +++++++---- src/allmydata/scripts/stats_gatherer.py | 9 ++++-- src/allmydata/scripts/tahoe_invite.py | 7 ++++- src/allmydata/scripts/types_.py | 10 +++++++ 8 files changed, 77 insertions(+), 36 deletions(-) create mode 100644 src/allmydata/scripts/types_.py diff --git a/src/allmydata/scripts/admin.py b/src/allmydata/scripts/admin.py index e472ffd8c..50dde9e43 100644 --- a/src/allmydata/scripts/admin.py +++ b/src/allmydata/scripts/admin.py @@ -1,5 +1,10 @@ from __future__ import print_function +try: + from allmydata.scripts.types_ import SubCommands +except ImportError: + pass + from twisted.python import usage from allmydata.scripts.common import BaseOptions @@ -79,8 +84,8 @@ def do_admin(options): subCommands = [ - ["admin", None, AdminCommand, "admin subcommands: use 'tahoe admin' for a list"], - ] + ("admin", None, AdminCommand, "admin subcommands: use 'tahoe admin' for a list"), + ] # type: SubCommands dispatch = { "admin": do_admin, diff --git a/src/allmydata/scripts/cli.py b/src/allmydata/scripts/cli.py index 96dde65f8..2bb91472f 100644 --- a/src/allmydata/scripts/cli.py +++ b/src/allmydata/scripts/cli.py @@ -4,6 +4,7 @@ import os.path, re, fnmatch try: from typing import List, Sequence, Any + from allmydata.scripts.types_ import SubCommands except ImportError: pass @@ -461,25 +462,25 @@ class DeepCheckOptions(FileStoreOptions): Optionally repair any problems found.""" subCommands = [ - ["mkdir", None, MakeDirectoryOptions, "Create a new directory."], - ["add-alias", None, AddAliasOptions, "Add a new alias cap."], - ["create-alias", None, CreateAliasOptions, "Create a new alias cap."], - ["list-aliases", None, ListAliasesOptions, "List all alias caps."], - ["ls", None, ListOptions, "List a directory."], - ["get", None, GetOptions, "Retrieve a file from the grid."], - ["put", None, PutOptions, "Upload a file into the grid."], - ["cp", None, CpOptions, "Copy one or more files or directories."], - ["unlink", None, UnlinkOptions, "Unlink a file or directory on the grid."], - ["mv", None, MvOptions, "Move a file within the grid."], - ["ln", None, LnOptions, "Make an additional link to an existing file or directory."], - ["backup", None, BackupOptions, "Make target dir look like local dir."], - ["webopen", None, WebopenOptions, "Open a web browser to a grid file or directory."], - ["manifest", None, ManifestOptions, "List all files/directories in a subtree."], - ["stats", None, StatsOptions, "Print statistics about all files/directories in a subtree."], - ["check", None, CheckOptions, "Check a single file or directory."], - ["deep-check", None, DeepCheckOptions, "Check all files/directories reachable from a starting point."], - ["status", None, TahoeStatusCommand, "Various status information."], - ] + ("mkdir", None, MakeDirectoryOptions, "Create a new directory."), + ("add-alias", None, AddAliasOptions, "Add a new alias cap."), + ("create-alias", None, CreateAliasOptions, "Create a new alias cap."), + ("list-aliases", None, ListAliasesOptions, "List all alias caps."), + ("ls", None, ListOptions, "List a directory."), + ("get", None, GetOptions, "Retrieve a file from the grid."), + ("put", None, PutOptions, "Upload a file into the grid."), + ("cp", None, CpOptions, "Copy one or more files or directories."), + ("unlink", None, UnlinkOptions, "Unlink a file or directory on the grid."), + ("mv", None, MvOptions, "Move a file within the grid."), + ("ln", None, LnOptions, "Make an additional link to an existing file or directory."), + ("backup", None, BackupOptions, "Make target dir look like local dir."), + ("webopen", None, WebopenOptions, "Open a web browser to a grid file or directory."), + ("manifest", None, ManifestOptions, "List all files/directories in a subtree."), + ("stats", None, StatsOptions, "Print statistics about all files/directories in a subtree."), + ("check", None, CheckOptions, "Check a single file or directory."), + ("deep-check", None, DeepCheckOptions, "Check all files/directories reachable from a starting point."), + ("status", None, TahoeStatusCommand, "Various status information."), + ] # type: SubCommands def mkdir(options): from allmydata.scripts import tahoe_mkdir diff --git a/src/allmydata/scripts/create_node.py b/src/allmydata/scripts/create_node.py index 2634e0915..9af9bf3ad 100644 --- a/src/allmydata/scripts/create_node.py +++ b/src/allmydata/scripts/create_node.py @@ -3,6 +3,11 @@ from __future__ import print_function import os import json +try: + from allmydata.scripts.types_ import SubCommands +except ImportError: + pass + from twisted.internet import reactor, defer from twisted.python.usage import UsageError from allmydata.scripts.common import BasedirOptions, NoDefaultBasedirOptions @@ -478,10 +483,10 @@ def create_introducer(config): subCommands = [ - ["create-node", None, CreateNodeOptions, "Create a node that acts as a client, server or both."], - ["create-client", None, CreateClientOptions, "Create a client node (with storage initially disabled)."], - ["create-introducer", None, CreateIntroducerOptions, "Create an introducer node."], -] + ("create-node", None, CreateNodeOptions, "Create a node that acts as a client, server or both."), + ("create-client", None, CreateClientOptions, "Create a client node (with storage initially disabled)."), + ("create-introducer", None, CreateIntroducerOptions, "Create an introducer node."), +] # type: SubCommands dispatch = { "create-node": create_node, diff --git a/src/allmydata/scripts/debug.py b/src/allmydata/scripts/debug.py index fd3f2b87c..be6b7e2bc 100644 --- a/src/allmydata/scripts/debug.py +++ b/src/allmydata/scripts/debug.py @@ -1,5 +1,10 @@ from __future__ import print_function +try: + from allmydata.scripts.types_ import SubCommands +except ImportError: + pass + # do not import any allmydata modules at this level. Do that from inside # individual functions instead. import struct, time, os, sys @@ -1051,8 +1056,8 @@ def do_debug(options): subCommands = [ - ["debug", None, DebugCommand, "debug subcommands: use 'tahoe debug' for a list."], - ] + ("debug", None, DebugCommand, "debug subcommands: use 'tahoe debug' for a list."), + ] # type: SubCommands dispatch = { "debug": do_debug, diff --git a/src/allmydata/scripts/runner.py b/src/allmydata/scripts/runner.py index 3436a1b84..999d7d353 100644 --- a/src/allmydata/scripts/runner.py +++ b/src/allmydata/scripts/runner.py @@ -4,6 +4,11 @@ import os, sys from six.moves import StringIO import six +try: + from allmydata.scripts.types_ import SubCommands +except ImportError: + pass + from twisted.python import usage from twisted.internet import defer, task, threads @@ -45,12 +50,12 @@ _control_node_dispatch = { } process_control_commands = [ - ["run", None, tahoe_run.RunOptions, "run a node without daemonizing"], - ["daemonize", None, tahoe_daemonize.DaemonizeOptions, "(deprecated) run a node in the background"], - ["start", None, tahoe_start.StartOptions, "(deprecated) start a node in the background and confirm it started"], - ["stop", None, tahoe_stop.StopOptions, "(deprecated) stop a node"], - ["restart", None, tahoe_restart.RestartOptions, "(deprecated) restart a node"], -] + ("run", None, tahoe_run.RunOptions, "run a node without daemonizing"), + ("daemonize", None, tahoe_daemonize.DaemonizeOptions, "(deprecated) run a node in the background"), + ("start", None, tahoe_start.StartOptions, "(deprecated) start a node in the background and confirm it started"), + ("stop", None, tahoe_stop.StopOptions, "(deprecated) stop a node"), + ("restart", None, tahoe_restart.RestartOptions, "(deprecated) restart a node"), +] # type: SubCommands class Options(usage.Options): diff --git a/src/allmydata/scripts/stats_gatherer.py b/src/allmydata/scripts/stats_gatherer.py index 26848a23c..b16ce689e 100644 --- a/src/allmydata/scripts/stats_gatherer.py +++ b/src/allmydata/scripts/stats_gatherer.py @@ -2,6 +2,11 @@ from __future__ import print_function import os +try: + from allmydata.scripts.types_ import SubCommands +except ImportError: + pass + # Python 2 compatibility from future.utils import PY2 if PY2: @@ -93,8 +98,8 @@ def create_stats_gatherer(config): return 0 subCommands = [ - ["create-stats-gatherer", None, CreateStatsGathererOptions, "Create a stats-gatherer service."], -] + ("create-stats-gatherer", None, CreateStatsGathererOptions, "Create a stats-gatherer service."), +] # type: SubCommands dispatch = { "create-stats-gatherer": create_stats_gatherer, diff --git a/src/allmydata/scripts/tahoe_invite.py b/src/allmydata/scripts/tahoe_invite.py index cca4216e3..f2d978f55 100644 --- a/src/allmydata/scripts/tahoe_invite.py +++ b/src/allmydata/scripts/tahoe_invite.py @@ -3,6 +3,11 @@ from __future__ import print_function import json from os.path import join +try: + from allmydata.scripts.types_ import SubCommands +except ImportError: + pass + from twisted.python import usage from twisted.internet import defer, reactor @@ -104,7 +109,7 @@ def invite(options): subCommands = [ ("invite", None, InviteOptions, "Invite a new node to this grid"), -] +] # type: SubCommands dispatch = { "invite": invite, diff --git a/src/allmydata/scripts/types_.py b/src/allmydata/scripts/types_.py new file mode 100644 index 000000000..58f88722b --- /dev/null +++ b/src/allmydata/scripts/types_.py @@ -0,0 +1,10 @@ +from typing import List, Tuple, Type +from allmydata.scripts.common import BaseOptions + + +# Historically, subcommands were implemented as lists, but due to a +# [designed contraint in mypy](https://stackoverflow.com/a/52559625/70170), +# a Tuple is required. +SubCommand = Tuple[str, None, Type[BaseOptions], str] + +SubCommands = List[SubCommand] From 572d7b2e02b2ba54c8c4db1592bac1898b48aba1 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 29 Nov 2020 15:52:04 -0500 Subject: [PATCH 38/81] Ignore error when untyped Module has no dispatch. --- src/allmydata/scripts/runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/scripts/runner.py b/src/allmydata/scripts/runner.py index 999d7d353..e56130d87 100644 --- a/src/allmydata/scripts/runner.py +++ b/src/allmydata/scripts/runner.py @@ -113,7 +113,7 @@ class Options(usage.Options): create_dispatch = {} for module in (create_node, stats_gatherer): - create_dispatch.update(module.dispatch) + create_dispatch.update(module.dispatch) # type: ignore def parse_options(argv, config=None): if not config: From b1b3a2341517d2b993e21f6764797aa3f6c01d4f Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 29 Nov 2020 15:56:39 -0500 Subject: [PATCH 39/81] Fix type errors with CPUUsageMonitor subclasses with float POLL_INTERVAL. --- src/allmydata/stats.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/stats.py b/src/allmydata/stats.py index f669b0861..6de323b73 100644 --- a/src/allmydata/stats.py +++ b/src/allmydata/stats.py @@ -78,7 +78,7 @@ class LoadMonitor(service.MultiService): @implementer(IStatsProducer) class CPUUsageMonitor(service.MultiService): HISTORY_LENGTH = 15 - POLL_INTERVAL = 60 + POLL_INTERVAL = 60 # type: float def __init__(self): service.MultiService.__init__(self) From 1768377aecc4d7e563e0b42d0c79d8d0f2d7d8a5 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 29 Nov 2020 16:04:56 -0500 Subject: [PATCH 40/81] Ignore error in DummyStorage --- src/allmydata/test/storage_plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/storage_plugin.py b/src/allmydata/test/storage_plugin.py index 4a1f84531..24081ae09 100644 --- a/src/allmydata/test/storage_plugin.py +++ b/src/allmydata/test/storage_plugin.py @@ -48,7 +48,7 @@ class RIDummy(RemoteInterface): -@implementer(IFoolscapStoragePlugin) +@implementer(IFoolscapStoragePlugin) # type: ignore # todo: make stubs for twisted @attr.s class DummyStorage(object): name = attr.ib() From 3653d7ed16c6b5d365664aca504af0c388e99752 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 29 Nov 2020 16:05:09 -0500 Subject: [PATCH 41/81] Ignore type checks on Referenceable objects. Ref warner/foolscap#78. --- src/allmydata/test/storage_plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/storage_plugin.py b/src/allmydata/test/storage_plugin.py index 24081ae09..b3464a88f 100644 --- a/src/allmydata/test/storage_plugin.py +++ b/src/allmydata/test/storage_plugin.py @@ -107,7 +107,7 @@ class GetCounter(Resource, object): @implementer(RIDummy) @attr.s(frozen=True) -class DummyStorageServer(object): +class DummyStorageServer(object): # type: ignore # warner/foolscap#78 get_anonymous_storage_server = attr.ib() def remote_just_some_method(self): From 2514196b275ad65b9e546b4c54898a43be949edd Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 29 Nov 2020 16:06:46 -0500 Subject: [PATCH 42/81] Suppress typing error in DummyStorageClient --- src/allmydata/test/storage_plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/storage_plugin.py b/src/allmydata/test/storage_plugin.py index b3464a88f..ba11776e6 100644 --- a/src/allmydata/test/storage_plugin.py +++ b/src/allmydata/test/storage_plugin.py @@ -116,7 +116,7 @@ class DummyStorageServer(object): # type: ignore # warner/foolscap#78 @implementer(IStorageServer) @attr.s -class DummyStorageClient(object): +class DummyStorageClient(object): # type: ignore # incomplete implementation get_rref = attr.ib() configuration = attr.ib() announcement = attr.ib() From ffa19d1c07e6a876fcebe8e8edbe968b10c842a9 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 29 Nov 2020 16:08:22 -0500 Subject: [PATCH 43/81] Suppress typing errors in common Nodes --- src/allmydata/test/common.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/common.py b/src/allmydata/test/common.py index a420dd3ba..a68a56ba5 100644 --- a/src/allmydata/test/common.py +++ b/src/allmydata/test/common.py @@ -391,7 +391,7 @@ class DummyProducer(object): pass @implementer(IImmutableFileNode) -class FakeCHKFileNode(object): +class FakeCHKFileNode(object): # type: ignore # incomplete implementation """I provide IImmutableFileNode, but all of my data is stored in a class-level dictionary.""" @@ -529,7 +529,7 @@ def create_chk_filenode(contents, all_contents): @implementer(IMutableFileNode, ICheckable) -class FakeMutableFileNode(object): +class FakeMutableFileNode(object): # type: ignore # incomplete implementation """I provide IMutableFileNode, but all of my data is stored in a class-level dictionary.""" From 53ff740f0ec2a13b50673fb989190cc73bee60f6 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 29 Nov 2020 16:14:10 -0500 Subject: [PATCH 44/81] Suppress type check error on NativeStorageServerWithVersion --- src/allmydata/test/test_storage_client.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/allmydata/test/test_storage_client.py b/src/allmydata/test/test_storage_client.py index fa3a34b15..f2be9ad1e 100644 --- a/src/allmydata/test/test_storage_client.py +++ b/src/allmydata/test/test_storage_client.py @@ -90,7 +90,12 @@ from allmydata.interfaces import ( SOME_FURL = b"pb://abcde@nowhere/fake" -class NativeStorageServerWithVersion(NativeStorageServer): + +# type checks fail with: +# Cannot determine consistent method resolution order (MRO) for "NativeStorageServerWithVersion" +# even though class hierarchy is single-inheritance. Probably `implementer` +# wrappers are affecting the MRO. +class NativeStorageServerWithVersion(NativeStorageServer): # type: ignore def __init__(self, version): # note: these instances won't work for anything other than # get_available_space() because we don't upcall From 7507e84a18902f5874a9aaaabf45dee92ec05510 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 29 Nov 2020 16:19:24 -0500 Subject: [PATCH 45/81] Suppress errors in no_network --- src/allmydata/test/no_network.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/allmydata/test/no_network.py b/src/allmydata/test/no_network.py index 59ab807bb..1e8c519e1 100644 --- a/src/allmydata/test/no_network.py +++ b/src/allmydata/test/no_network.py @@ -67,7 +67,7 @@ class Marker(object): fireNow = partial(defer.succeed, None) -@implementer(IRemoteReference) +@implementer(IRemoteReference) # type: ignore # todo: write stubs for foolscap class LocalWrapper(object): """ A ``LocalWrapper`` presents the remote reference interface to a local @@ -212,9 +212,12 @@ class NoNetworkServer(object): return _StorageServer(lambda: self.rref) def get_version(self): return self.rref.version + def start_connecting(self, trigger_cb): + raise NotImplementedError + @implementer(IStorageBroker) -class NoNetworkStorageBroker(object): +class NoNetworkStorageBroker(object): # type: ignore # missing many methods def get_servers_for_psi(self, peer_selection_index): def _permuted(server): seed = server.get_permutation_seed() @@ -258,7 +261,7 @@ def create_no_network_client(basedir): return defer.succeed(client) -class _NoNetworkClient(_Client): +class _NoNetworkClient(_Client): # type: ignore # Cannot determine consistent MRO order """ Overrides all _Client networking functionality to do nothing. """ From 13cd780231bf3185104d930d748c75b9d324eabc Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 29 Nov 2020 16:22:12 -0500 Subject: [PATCH 46/81] Prefer sys.maxsize to sys.maxint. --- src/allmydata/test/check_memory.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/allmydata/test/check_memory.py b/src/allmydata/test/check_memory.py index 41cf6e1d7..4f50b383b 100644 --- a/src/allmydata/test/check_memory.py +++ b/src/allmydata/test/check_memory.py @@ -499,13 +499,13 @@ if __name__ == '__main__': mode = "upload" if len(sys.argv) > 1: mode = sys.argv[1] - if sys.maxint == 2147483647: + if sys.maxsize == 2147483647: bits = "32" - elif sys.maxint == 9223372036854775807: + elif sys.maxsize == 9223372036854775807: bits = "64" else: bits = "?" - print("%s-bit system (sys.maxint=%d)" % (bits, sys.maxint)) + print("%s-bit system (sys.maxsize=%d)" % (bits, sys.maxsize)) # put the logfile and stats.out in _test_memory/ . These stick around. # put the nodes and other files in _test_memory/test/ . These are # removed each time we run. From b0803a2ac054234f341ec1650695ce988d926f25 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 29 Nov 2020 16:24:20 -0500 Subject: [PATCH 47/81] Suppress errors in test_web due to ambiguous MRO --- src/allmydata/test/web/test_web.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/web/test_web.py b/src/allmydata/test/web/test_web.py index 326569a26..a5fbe5a51 100644 --- a/src/allmydata/test/web/test_web.py +++ b/src/allmydata/test/web/test_web.py @@ -189,7 +189,7 @@ class FakeHistory(object): def list_all_helper_statuses(self): return [] -class FakeDisplayableServer(StubServer): +class FakeDisplayableServer(StubServer): # type: ignore # Cannot determine MRO def __init__(self, serverid, nickname, connected, last_connect_time, last_loss_time, last_rx_time): StubServer.__init__(self, serverid) @@ -255,7 +255,7 @@ class FakeStorageServer(service.MultiService): def on_status_changed(self, cb): cb(self) -class FakeClient(_Client): +class FakeClient(_Client): # type: ignore # Cannot determine MRO def __init__(self): # don't upcall to Client.__init__, since we only want to initialize a # minimal subset From fc19d1baf4a503dabc1a69b93adfd3200b5e413b Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 29 Nov 2020 16:26:36 -0500 Subject: [PATCH 48/81] Suppress errors in test_sftp --- src/allmydata/test/test_sftp.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/allmydata/test/test_sftp.py b/src/allmydata/test/test_sftp.py index b6f1fbc8a..1ff0363e8 100644 --- a/src/allmydata/test/test_sftp.py +++ b/src/allmydata/test/test_sftp.py @@ -9,18 +9,15 @@ from twisted.python.failure import Failure from twisted.internet.error import ProcessDone, ProcessTerminated from allmydata.util import deferredutil -conch_interfaces = None -sftp = None -sftpd = None - try: from twisted.conch import interfaces as conch_interfaces from twisted.conch.ssh import filetransfer as sftp from allmydata.frontends import sftpd except ImportError as e: + conch_interfaces = sftp = sftpd = None # type: ignore conch_unavailable_reason = e else: - conch_unavailable_reason = None + conch_unavailable_reason = None # type: ignore from allmydata.interfaces import IDirectoryNode, ExistingChildError, NoSuchChildError from allmydata.mutable.common import NotWriteableError From 54e45498367485d44ed6f886c50feb269a26c9e5 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 29 Nov 2020 16:30:13 -0500 Subject: [PATCH 49/81] Satisfy type check in test_helper.FakeClient. --- src/allmydata/test/test_helper.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/allmydata/test/test_helper.py b/src/allmydata/test/test_helper.py index 65c07135a..3faffbe0d 100644 --- a/src/allmydata/test/test_helper.py +++ b/src/allmydata/test/test_helper.py @@ -19,6 +19,12 @@ from functools import ( ) import attr +try: + from typing import List + from allmydata.introducer.client import IntroducerClient +except ImportError: + pass + from twisted.internet import defer from twisted.trial import unittest from twisted.application import service @@ -125,7 +131,7 @@ class FakeCHKCheckerAndUEBFetcher(object): )) class FakeClient(service.MultiService): - introducer_clients = [] + introducer_clients = [] # type: List[IntroducerClient] DEFAULT_ENCODING_PARAMETERS = {"k":25, "happy": 75, "n": 100, From 86f88a4aa5568876989f1bc222703cc9dd3d35fb Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 29 Nov 2020 16:32:19 -0500 Subject: [PATCH 50/81] Satisfy type checks in test_dirnode --- src/allmydata/test/test_dirnode.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/test_dirnode.py b/src/allmydata/test/test_dirnode.py index 48ffff45a..8af1567c9 100644 --- a/src/allmydata/test/test_dirnode.py +++ b/src/allmydata/test/test_dirnode.py @@ -1526,7 +1526,7 @@ class Packing(testutil.ReallyEqualMixin, unittest.TestCase): kids, fn.get_writekey(), deep_immutable=True) @implementer(IMutableFileNode) -class FakeMutableFile(object): +class FakeMutableFile(object): # type: ignore # incomplete interface counter = 0 def __init__(self, initial_contents=""): data = self._get_initial_contents(initial_contents) @@ -1587,7 +1587,7 @@ class FakeNodeMaker(NodeMaker): def create_mutable_file(self, contents="", keysize=None, version=None): return defer.succeed(FakeMutableFile(contents)) -class FakeClient2(_Client): +class FakeClient2(_Client): # type: ignore # ambiguous MRO def __init__(self): self.nodemaker = FakeNodeMaker(None, None, None, None, None, From d2d3f1f4a9a6aecb482859c96e5732c41847051f Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 29 Nov 2020 16:33:25 -0500 Subject: [PATCH 51/81] Suppress type errors in test_checker --- src/allmydata/test/test_checker.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/allmydata/test/test_checker.py b/src/allmydata/test/test_checker.py index 85b894b1f..23ce44762 100644 --- a/src/allmydata/test/test_checker.py +++ b/src/allmydata/test/test_checker.py @@ -62,7 +62,7 @@ class FakeClient(object): @implementer(IServer) -class FakeServer(object): +class FakeServer(object): # type: ignore # incomplete interface def get_name(self): return "fake name" @@ -75,7 +75,7 @@ class FakeServer(object): @implementer(ICheckResults) -class FakeCheckResults(object): +class FakeCheckResults(object): # type: ignore # incomplete interface def __init__(self, si=None, healthy=False, recoverable=False, @@ -106,7 +106,7 @@ class FakeCheckResults(object): @implementer(ICheckAndRepairResults) -class FakeCheckAndRepairResults(object): +class FakeCheckAndRepairResults(object): # type: ignore # incomplete interface def __init__(self, si=None, repair_attempted=False, From d2e2a22f6230f880c133f05cad73bd9b8490a909 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 4 Dec 2020 10:34:21 -0500 Subject: [PATCH 52/81] Run typechecks in CircleCI --- .circleci/config.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index afa3fafa1..ff14d6dd3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -91,6 +91,9 @@ workflows: - "build-porting-depgraph": <<: *DOCKERHUB_CONTEXT + - "typechecks": + <<: *DOCKERHUB_CONTEXT + images: # Build the Docker images used by the ci jobs. This makes the ci jobs # faster and takes various spurious failures out of the critical path. @@ -475,6 +478,17 @@ jobs: . /tmp/venv/bin/activate ./misc/python3/depgraph.sh + typechecks: + docker: + - <<: *DOCKERHUB_AUTH + image: "jaraco/multipy-tox" + + steps: + - "checkout" + - run: + name: "Validate Types" + command: tox -e typechecks + build-image: &BUILD_IMAGE # This is a template for a job to build a Docker image that has as much of # the setup as we can manage already done and baked in. This cuts down on From 3eb975748a9a6c32c7972ce4c498ec5afabfda9d Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 4 Dec 2020 19:56:51 -0500 Subject: [PATCH 53/81] Ignore type checks in allmydata. --- src/allmydata/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/__init__.py b/src/allmydata/__init__.py index 15d5fb240..a43781158 100644 --- a/src/allmydata/__init__.py +++ b/src/allmydata/__init__.py @@ -14,7 +14,7 @@ __all__ = [ __version__ = "unknown" try: - from allmydata._version import __version__ + from allmydata._version import __version__ # type: ignore except ImportError: # We're running in a tree that hasn't run update_version, and didn't # come with a _version.py, so we don't know what our version is. @@ -24,7 +24,7 @@ except ImportError: full_version = "unknown" branch = "unknown" try: - from allmydata._version import full_version, branch + from allmydata._version import full_version, branch # type: ignore except ImportError: # We're running in a tree that hasn't run update_version, and didn't # come with a _version.py, so we don't know what our full version or From 950ca189326703daf8bb84188507ca7324efc16e Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 18 Dec 2020 09:23:27 -0500 Subject: [PATCH 54/81] Ignores no longer needed. --- src/allmydata/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/__init__.py b/src/allmydata/__init__.py index a43781158..15d5fb240 100644 --- a/src/allmydata/__init__.py +++ b/src/allmydata/__init__.py @@ -14,7 +14,7 @@ __all__ = [ __version__ = "unknown" try: - from allmydata._version import __version__ # type: ignore + from allmydata._version import __version__ except ImportError: # We're running in a tree that hasn't run update_version, and didn't # come with a _version.py, so we don't know what our version is. @@ -24,7 +24,7 @@ except ImportError: full_version = "unknown" branch = "unknown" try: - from allmydata._version import full_version, branch # type: ignore + from allmydata._version import full_version, branch except ImportError: # We're running in a tree that hasn't run update_version, and didn't # come with a _version.py, so we don't know what our full version or From 99da74fffb1b013602ae847dfd4756ccb5e61c3d Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 18 Dec 2020 09:58:56 -0500 Subject: [PATCH 55/81] Change comment to clarify that it's the implementation that's incomplete. --- src/allmydata/test/test_checker.py | 6 +++--- src/allmydata/test/test_dirnode.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/allmydata/test/test_checker.py b/src/allmydata/test/test_checker.py index 23ce44762..d24ad4ef4 100644 --- a/src/allmydata/test/test_checker.py +++ b/src/allmydata/test/test_checker.py @@ -62,7 +62,7 @@ class FakeClient(object): @implementer(IServer) -class FakeServer(object): # type: ignore # incomplete interface +class FakeServer(object): # type: ignore # incomplete implementation def get_name(self): return "fake name" @@ -75,7 +75,7 @@ class FakeServer(object): # type: ignore # incomplete interface @implementer(ICheckResults) -class FakeCheckResults(object): # type: ignore # incomplete interface +class FakeCheckResults(object): # type: ignore # incomplete implementation def __init__(self, si=None, healthy=False, recoverable=False, @@ -106,7 +106,7 @@ class FakeCheckResults(object): # type: ignore # incomplete interface @implementer(ICheckAndRepairResults) -class FakeCheckAndRepairResults(object): # type: ignore # incomplete interface +class FakeCheckAndRepairResults(object): # type: ignore # incomplete implementation def __init__(self, si=None, repair_attempted=False, diff --git a/src/allmydata/test/test_dirnode.py b/src/allmydata/test/test_dirnode.py index 8af1567c9..68fcdd54b 100644 --- a/src/allmydata/test/test_dirnode.py +++ b/src/allmydata/test/test_dirnode.py @@ -1526,7 +1526,7 @@ class Packing(testutil.ReallyEqualMixin, unittest.TestCase): kids, fn.get_writekey(), deep_immutable=True) @implementer(IMutableFileNode) -class FakeMutableFile(object): # type: ignore # incomplete interface +class FakeMutableFile(object): # type: ignore # incomplete implementation counter = 0 def __init__(self, initial_contents=""): data = self._get_initial_contents(initial_contents) From 51b0b201b49733280702ed901baa06e23d75af85 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 18 Dec 2020 10:33:30 -0500 Subject: [PATCH 56/81] Expand comment to provide more context. --- src/allmydata/web/private.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/allmydata/web/private.py b/src/allmydata/web/private.py index a86c869c4..405ca75e7 100644 --- a/src/allmydata/web/private.py +++ b/src/allmydata/web/private.py @@ -61,7 +61,12 @@ class IToken(ICredentials): pass -# Shoobx/mypy-zope#26 +# Workaround for Shoobx/mypy-zope#26, where without suitable +# stubs for twisted classes (ICredentials), IToken does not +# appear to be an Interface. The proper fix appears to be to +# create stubs for twisted +# (https://twistedmatrix.com/trac/ticket/9717). For now, +# bypassing the inline decorator syntax works around the issue. _itoken_impl = implementer(IToken) From efd0aef2847814c1c43a14db3ccc1cde01a95119 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 18 Dec 2020 10:36:43 -0500 Subject: [PATCH 57/81] Indicate that unichr is Python 2 only. --- src/allmydata/windows/fixups.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/windows/fixups.py b/src/allmydata/windows/fixups.py index c5ba3bb57..e98aa8a67 100644 --- a/src/allmydata/windows/fixups.py +++ b/src/allmydata/windows/fixups.py @@ -219,7 +219,7 @@ def initialize(): def unmangle(s): return re.sub( u'\\x7F[0-9a-fA-F]*\\;', - # type ignored for 'unichr' + # type ignored for 'unichr' (Python 2 only) lambda m: unichr(int(m.group(0)[1:-1], 16)), # type: ignore s, ) From ea0c10ef83d98f019472c744ad81f3c5a8f5fab1 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 18 Dec 2020 10:51:56 -0500 Subject: [PATCH 58/81] Remove set_size, unused --- src/allmydata/immutable/encode.py | 3 --- src/allmydata/interfaces.py | 5 ----- 2 files changed, 8 deletions(-) diff --git a/src/allmydata/immutable/encode.py b/src/allmydata/immutable/encode.py index e743ce766..9351df501 100644 --- a/src/allmydata/immutable/encode.py +++ b/src/allmydata/immutable/encode.py @@ -711,6 +711,3 @@ class Encoder(object): return self.uri_extension_data def get_uri_extension_hash(self): return self.uri_extension_hash - - def set_size(self, size): - raise NotImplementedError() diff --git a/src/allmydata/interfaces.py b/src/allmydata/interfaces.py index 95b1fdf63..edd16ea08 100644 --- a/src/allmydata/interfaces.py +++ b/src/allmydata/interfaces.py @@ -1825,11 +1825,6 @@ class IEncoder(Interface): willing to receive data. """ - def set_size(size): - """Specify the number of bytes that will be encoded. This must be - peformed before get_serialized_params() can be called. - """ - def set_encrypted_uploadable(u): """Provide a source of encrypted upload data. 'u' must implement IEncryptedUploadable. From d051791e9530e5e64d49942cdfd6afc0a9a98c17 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 18 Dec 2020 10:53:24 -0500 Subject: [PATCH 59/81] Add reference to ticket. --- src/allmydata/immutable/offloaded.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/allmydata/immutable/offloaded.py b/src/allmydata/immutable/offloaded.py index 53a1f911a..2d2c5c1f5 100644 --- a/src/allmydata/immutable/offloaded.py +++ b/src/allmydata/immutable/offloaded.py @@ -499,6 +499,7 @@ class LocalCiphertextReader(AskUntilSuccessMixin): # ??. I'm not sure if it makes sense to forward the close message. return self.call("close") + # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3561 def set_upload_status(self, upload_status): raise NotImplementedError From 090031cbfc33510e7eebc627f313c94c548ea91e Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 18 Dec 2020 10:55:20 -0500 Subject: [PATCH 60/81] Remove confirm_share_allocation from interface (unused). --- src/allmydata/immutable/upload.py | 3 --- src/allmydata/interfaces.py | 6 ------ 2 files changed, 9 deletions(-) diff --git a/src/allmydata/immutable/upload.py b/src/allmydata/immutable/upload.py index 171b71eff..e6da4812a 100644 --- a/src/allmydata/immutable/upload.py +++ b/src/allmydata/immutable/upload.py @@ -388,9 +388,6 @@ class PeerSelector(object): def add_peers(self, peerids=None): raise NotImplementedError - def confirm_share_allocation(self, peerid, shnum): - raise NotImplementedError - class _QueryStatistics(object): diff --git a/src/allmydata/interfaces.py b/src/allmydata/interfaces.py index edd16ea08..e460854f8 100644 --- a/src/allmydata/interfaces.py +++ b/src/allmydata/interfaces.py @@ -860,12 +860,6 @@ class IPeerSelector(Interface): peer selection begins. """ - def confirm_share_allocation(peerid, shnum): - """ - Confirm that an allocated peer=>share pairing has been - successfully established. - """ - def add_peers(peerids=set): """ Update my internal state to include the peers in peerids as From 0e248cb4ef03f40da7e29c1bcd8a1c390ea00105 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 18 Dec 2020 10:59:53 -0500 Subject: [PATCH 61/81] Declare signing key as required in introducer client publish. --- src/allmydata/introducer/client.py | 2 +- src/allmydata/introducer/interfaces.py | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/allmydata/introducer/client.py b/src/allmydata/introducer/client.py index 62642d0af..0e0a56442 100644 --- a/src/allmydata/introducer/client.py +++ b/src/allmydata/introducer/client.py @@ -198,7 +198,7 @@ class IntroducerClient(service.Service, Referenceable): ann_d.update(ann) return ann_d - def publish(self, service_name, ann, signing_key=None): + def publish(self, service_name, ann, signing_key): # we increment the seqnum every time we publish something new current_seqnum, current_nonce = self._sequencer() diff --git a/src/allmydata/introducer/interfaces.py b/src/allmydata/introducer/interfaces.py index 9f08f1943..24fd3945f 100644 --- a/src/allmydata/introducer/interfaces.py +++ b/src/allmydata/introducer/interfaces.py @@ -73,7 +73,7 @@ class IIntroducerClient(Interface): publish their services to the rest of the world, and I help them learn about services available on other nodes.""" - def publish(service_name, ann, signing_key=None): + def publish(service_name, ann, signing_key): """Publish the given announcement dictionary (which must be JSON-serializable), plus some additional keys, to the world. @@ -83,8 +83,7 @@ class IIntroducerClient(Interface): the signing_key, if present, otherwise it is derived from the 'anonymous-storage-FURL' key. - If signing_key= is set to an instance of SigningKey, it will be - used to sign the announcement.""" + signing_key (a SigningKey) will be used to sign the announcement.""" def subscribe_to(service_name, callback, *args, **kwargs): """Call this if you will eventually want to use services with the From c2d2aba83f025824e23f2051700ad2fd54063799 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 18 Dec 2020 11:05:55 -0500 Subject: [PATCH 62/81] Add reference to ticket. --- src/allmydata/mutable/filenode.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/allmydata/mutable/filenode.py b/src/allmydata/mutable/filenode.py index 4613a918b..39e8b76be 100644 --- a/src/allmydata/mutable/filenode.py +++ b/src/allmydata/mutable/filenode.py @@ -1206,5 +1206,6 @@ class MutableFileVersion(object): mode=mode) return u.update() + # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3562 def get_servermap(self): raise NotImplementedError From 189608e11388988b512aa00f225911ff61afc80c Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 18 Dec 2020 11:07:30 -0500 Subject: [PATCH 63/81] Remove GENERATED_FILES, unused --- src/allmydata/node.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/allmydata/node.py b/src/allmydata/node.py index 7622d5bc3..6d0b2da04 100644 --- a/src/allmydata/node.py +++ b/src/allmydata/node.py @@ -713,7 +713,6 @@ class Node(service.MultiService): """ NODETYPE = "unknown NODETYPE" CERTFILE = "node.pem" - GENERATED_FILES = [] # type: ignore def __init__(self, config, main_tub, control_tub, i2p_provider, tor_provider): """ From 602a06e5cba818cb927c45566a610eb3e0d6eadc Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 18 Dec 2020 11:14:07 -0500 Subject: [PATCH 64/81] Extract Parameters type in scripts.types_. --- src/allmydata/scripts/cli.py | 5 ++--- src/allmydata/scripts/common.py | 7 ++++--- src/allmydata/scripts/types_.py | 4 +++- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/src/allmydata/scripts/cli.py b/src/allmydata/scripts/cli.py index 2bb91472f..e4cd8aa22 100644 --- a/src/allmydata/scripts/cli.py +++ b/src/allmydata/scripts/cli.py @@ -3,8 +3,7 @@ from __future__ import print_function import os.path, re, fnmatch try: - from typing import List, Sequence, Any - from allmydata.scripts.types_ import SubCommands + from allmydata.scripts.types_ import SubCommands, Parameters except ImportError: pass @@ -26,7 +25,7 @@ class FileStoreOptions(BaseOptions): "This overrides the URL found in the --node-directory ."], ["dir-cap", None, None, "Specify which dirnode URI should be used as the 'tahoe' alias."] - ] # type: List[Sequence[Any]] + ] # type: Parameters def postOptions(self): self["quiet"] = self.parent["quiet"] diff --git a/src/allmydata/scripts/common.py b/src/allmydata/scripts/common.py index b501c7e6a..42c26bb90 100644 --- a/src/allmydata/scripts/common.py +++ b/src/allmydata/scripts/common.py @@ -5,7 +5,8 @@ import codecs from os.path import join try: - from typing import Optional, Sequence, List, Any + from typing import Optional + from .types_ import Parameters except ImportError: pass @@ -69,7 +70,7 @@ class BasedirOptions(BaseOptions): optParameters = [ ["basedir", "C", None, "Specify which Tahoe base directory should be used. [default: %s]" % quote_local_unicode_path(_default_nodedir)], - ] # type: List[Sequence[Any]] + ] # type: Parameters def parseArgs(self, basedir=None): # This finds the node-directory option correctly even if we are in a subcommand. @@ -106,7 +107,7 @@ class NoDefaultBasedirOptions(BasedirOptions): optParameters = [ ["basedir", "C", None, "Specify which Tahoe base directory should be used."], - ] # type: List[Sequence[Any]] + ] # type: Parameters # This is overridden in order to ensure we get a "Wrong number of arguments." # error when more than one argument is given. diff --git a/src/allmydata/scripts/types_.py b/src/allmydata/scripts/types_.py index 58f88722b..3937cb803 100644 --- a/src/allmydata/scripts/types_.py +++ b/src/allmydata/scripts/types_.py @@ -1,4 +1,4 @@ -from typing import List, Tuple, Type +from typing import List, Tuple, Type, Sequence, Any from allmydata.scripts.common import BaseOptions @@ -8,3 +8,5 @@ from allmydata.scripts.common import BaseOptions SubCommand = Tuple[str, None, Type[BaseOptions], str] SubCommands = List[SubCommand] + +Parameters = List[Sequence[Any]] From 6b6b8f8378f1e3da826a50d9dcbe95e63ec8faa1 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 26 Dec 2020 11:17:35 -0500 Subject: [PATCH 65/81] Push IURI implementers down to the classes that actually implement it. --- src/allmydata/uri.py | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/src/allmydata/uri.py b/src/allmydata/uri.py index d7f4782cd..ce7794dda 100644 --- a/src/allmydata/uri.py +++ b/src/allmydata/uri.py @@ -494,7 +494,6 @@ class MDMFVerifierURI(_BaseURI): return self -@implementer(IURI, IDirnodeURI) class _DirectoryBaseURI(_BaseURI): def __init__(self, filenode_uri=None): self._filenode_uri = filenode_uri @@ -540,14 +539,8 @@ class _DirectoryBaseURI(_BaseURI): def get_storage_index(self): return self._filenode_uri.get_storage_index() - def get_readonly(self): - raise NotImplementedError() - def is_readonly(self): - raise NotImplementedError() - - -@implementer(IDirectoryURI) +@implementer(IURI, IDirectoryURI) class DirectoryURI(_DirectoryBaseURI): BASE_STRING=b'URI:DIR2:' @@ -566,7 +559,7 @@ class DirectoryURI(_DirectoryBaseURI): return ReadonlyDirectoryURI(self._filenode_uri.get_readonly()) -@implementer(IReadonlyDirectoryURI) +@implementer(IURI, IReadonlyDirectoryURI) class ReadonlyDirectoryURI(_DirectoryBaseURI): BASE_STRING=b'URI:DIR2-RO:' @@ -585,6 +578,7 @@ class ReadonlyDirectoryURI(_DirectoryBaseURI): return self +@implementer(IURI, IDirnodeURI) class _ImmutableDirectoryBaseURI(_DirectoryBaseURI): def __init__(self, filenode_uri=None): if filenode_uri: @@ -622,7 +616,7 @@ class LiteralDirectoryURI(_ImmutableDirectoryBaseURI): return None -@implementer(IDirectoryURI) +@implementer(IURI, IDirectoryURI) class MDMFDirectoryURI(_DirectoryBaseURI): BASE_STRING=b'URI:DIR2-MDMF:' @@ -644,7 +638,7 @@ class MDMFDirectoryURI(_DirectoryBaseURI): return MDMFDirectoryURIVerifier(self._filenode_uri.get_verify_cap()) -@implementer(IReadonlyDirectoryURI) +@implementer(IURI, IReadonlyDirectoryURI) class ReadonlyMDMFDirectoryURI(_DirectoryBaseURI): BASE_STRING=b'URI:DIR2-MDMF-RO:' @@ -682,7 +676,7 @@ def wrap_dirnode_cap(filecap): raise AssertionError("cannot interpret as a directory cap: %s" % filecap.__class__) -@implementer(IVerifierURI) +@implementer(IURI, IVerifierURI) class MDMFDirectoryURIVerifier(_DirectoryBaseURI): BASE_STRING=b'URI:DIR2-MDMF-Verifier:' @@ -707,7 +701,7 @@ class MDMFDirectoryURIVerifier(_DirectoryBaseURI): return self -@implementer(IVerifierURI) +@implementer(IURI, IVerifierURI) class DirectoryURIVerifier(_DirectoryBaseURI): BASE_STRING=b'URI:DIR2-Verifier:' From b65ef3cee6c6922c70edac842e28985ac7c20990 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 26 Dec 2020 11:22:39 -0500 Subject: [PATCH 66/81] Revert "Ignores no longer needed." This reverts commit 950ca189326703daf8bb84188507ca7324efc16e. --- src/allmydata/__init__.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/allmydata/__init__.py b/src/allmydata/__init__.py index 15d5fb240..3157c8c80 100644 --- a/src/allmydata/__init__.py +++ b/src/allmydata/__init__.py @@ -14,7 +14,9 @@ __all__ = [ __version__ = "unknown" try: - from allmydata._version import __version__ + # type ignored as it fails in CI + # (https://app.circleci.com/pipelines/github/tahoe-lafs/tahoe-lafs/1647/workflows/60ae95d4-abe8-492c-8a03-1ad3b9e42ed3/jobs/40972) + from allmydata._version import __version__ # type: ignore except ImportError: # We're running in a tree that hasn't run update_version, and didn't # come with a _version.py, so we don't know what our version is. @@ -24,7 +26,9 @@ except ImportError: full_version = "unknown" branch = "unknown" try: - from allmydata._version import full_version, branch + # type ignored as it fails in CI + # (https://app.circleci.com/pipelines/github/tahoe-lafs/tahoe-lafs/1647/workflows/60ae95d4-abe8-492c-8a03-1ad3b9e42ed3/jobs/40972) + from allmydata._version import full_version, branch # type: ignore except ImportError: # We're running in a tree that hasn't run update_version, and didn't # come with a _version.py, so we don't know what our full version or From ab2c544efcaad629f2d49173bfdc82e8de94d944 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 26 Dec 2020 11:50:06 -0500 Subject: [PATCH 67/81] Restore IDirnodeURI --- src/allmydata/uri.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/allmydata/uri.py b/src/allmydata/uri.py index ce7794dda..51671b0ac 100644 --- a/src/allmydata/uri.py +++ b/src/allmydata/uri.py @@ -494,6 +494,7 @@ class MDMFVerifierURI(_BaseURI): return self +@implementer(IDirnodeURI) class _DirectoryBaseURI(_BaseURI): def __init__(self, filenode_uri=None): self._filenode_uri = filenode_uri From dacdf7f12da89e214eec9b0543b15ef450b27ae3 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 26 Dec 2020 11:56:13 -0500 Subject: [PATCH 68/81] Add more detail and link to upstream issue for Twisted stubs. --- src/allmydata/test/storage_plugin.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/storage_plugin.py b/src/allmydata/test/storage_plugin.py index ba11776e6..17ec89078 100644 --- a/src/allmydata/test/storage_plugin.py +++ b/src/allmydata/test/storage_plugin.py @@ -47,8 +47,9 @@ class RIDummy(RemoteInterface): """ - -@implementer(IFoolscapStoragePlugin) # type: ignore # todo: make stubs for twisted +# type ignored due to missing stubs for Twisted +# https://twistedmatrix.com/trac/ticket/9717 +@implementer(IFoolscapStoragePlugin) # type: ignore @attr.s class DummyStorage(object): name = attr.ib() From 5396f9f97eaf5a541c2334eaf8ebdb5a2d3484f5 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 26 Dec 2020 12:02:47 -0500 Subject: [PATCH 69/81] Replace fixme with reference to foolscap issue. --- src/allmydata/storage/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py index b7df702d5..5f2ef3ac2 100644 --- a/src/allmydata/storage/server.py +++ b/src/allmydata/storage/server.py @@ -581,7 +581,7 @@ class StorageServer(service.MultiService, Referenceable): for share in six.viewvalues(shares): share.add_or_renew_lease(lease_info) - def slot_testv_and_readv_and_writev( # type: ignore # fixme + def slot_testv_and_readv_and_writev( # type: ignore # warner/foolscap#78 self, storage_index, secrets, From 1bf71fd69010a0a075f66076156f0e951042b8dd Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 26 Dec 2020 12:12:39 -0500 Subject: [PATCH 70/81] Replace todo with a ticket. --- src/allmydata/test/no_network.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/no_network.py b/src/allmydata/test/no_network.py index 1e8c519e1..ba497f81a 100644 --- a/src/allmydata/test/no_network.py +++ b/src/allmydata/test/no_network.py @@ -67,7 +67,7 @@ class Marker(object): fireNow = partial(defer.succeed, None) -@implementer(IRemoteReference) # type: ignore # todo: write stubs for foolscap +@implementer(IRemoteReference) # type: ignore # warner/foolscap#79 class LocalWrapper(object): """ A ``LocalWrapper`` presents the remote reference interface to a local From 01147f4627a10ecb4db5ddcdcb4a17f5d839a627 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 26 Dec 2020 12:33:22 -0500 Subject: [PATCH 71/81] Add reference to ticket for ambiguous MRO --- src/allmydata/test/no_network.py | 2 +- src/allmydata/test/test_dirnode.py | 2 +- src/allmydata/test/test_storage_client.py | 6 +----- src/allmydata/test/web/test_web.py | 4 ++-- 4 files changed, 5 insertions(+), 9 deletions(-) diff --git a/src/allmydata/test/no_network.py b/src/allmydata/test/no_network.py index ba497f81a..b9c9ecaeb 100644 --- a/src/allmydata/test/no_network.py +++ b/src/allmydata/test/no_network.py @@ -261,7 +261,7 @@ def create_no_network_client(basedir): return defer.succeed(client) -class _NoNetworkClient(_Client): # type: ignore # Cannot determine consistent MRO order +class _NoNetworkClient(_Client): # type: ignore # tahoe-lafs/ticket/3573 """ Overrides all _Client networking functionality to do nothing. """ diff --git a/src/allmydata/test/test_dirnode.py b/src/allmydata/test/test_dirnode.py index 68fcdd54b..6866bc88e 100644 --- a/src/allmydata/test/test_dirnode.py +++ b/src/allmydata/test/test_dirnode.py @@ -1587,7 +1587,7 @@ class FakeNodeMaker(NodeMaker): def create_mutable_file(self, contents="", keysize=None, version=None): return defer.succeed(FakeMutableFile(contents)) -class FakeClient2(_Client): # type: ignore # ambiguous MRO +class FakeClient2(_Client): # type: ignore # tahoe-lafs/ticket/3573 def __init__(self): self.nodemaker = FakeNodeMaker(None, None, None, None, None, diff --git a/src/allmydata/test/test_storage_client.py b/src/allmydata/test/test_storage_client.py index f2be9ad1e..bcbfffa1e 100644 --- a/src/allmydata/test/test_storage_client.py +++ b/src/allmydata/test/test_storage_client.py @@ -91,11 +91,7 @@ from allmydata.interfaces import ( SOME_FURL = b"pb://abcde@nowhere/fake" -# type checks fail with: -# Cannot determine consistent method resolution order (MRO) for "NativeStorageServerWithVersion" -# even though class hierarchy is single-inheritance. Probably `implementer` -# wrappers are affecting the MRO. -class NativeStorageServerWithVersion(NativeStorageServer): # type: ignore +class NativeStorageServerWithVersion(NativeStorageServer): # type: ignore # tahoe-lafs/ticket/3573 def __init__(self, version): # note: these instances won't work for anything other than # get_available_space() because we don't upcall diff --git a/src/allmydata/test/web/test_web.py b/src/allmydata/test/web/test_web.py index a5fbe5a51..ef0446077 100644 --- a/src/allmydata/test/web/test_web.py +++ b/src/allmydata/test/web/test_web.py @@ -189,7 +189,7 @@ class FakeHistory(object): def list_all_helper_statuses(self): return [] -class FakeDisplayableServer(StubServer): # type: ignore # Cannot determine MRO +class FakeDisplayableServer(StubServer): # type: ignore # tahoe-lafs/ticket/3573 def __init__(self, serverid, nickname, connected, last_connect_time, last_loss_time, last_rx_time): StubServer.__init__(self, serverid) @@ -255,7 +255,7 @@ class FakeStorageServer(service.MultiService): def on_status_changed(self, cb): cb(self) -class FakeClient(_Client): # type: ignore # Cannot determine MRO +class FakeClient(_Client): # type: ignore # tahoe-lafs/ticket/3573 def __init__(self): # don't upcall to Client.__init__, since we only want to initialize a # minimal subset From 04ab4dec3be3fdd07f7a85bfe50c3133cef30910 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 26 Dec 2020 13:01:27 -0500 Subject: [PATCH 72/81] Extract function and annotate it to satisfy typechecks for _Config._basedir --- src/allmydata/node.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/allmydata/node.py b/src/allmydata/node.py index 89d5bcb90..fabb255f6 100644 --- a/src/allmydata/node.py +++ b/src/allmydata/node.py @@ -23,6 +23,11 @@ from base64 import b32decode, b32encode from errno import ENOENT, EPERM from warnings import warn +try: + from typing import Union +except ImportError: + pass + import attr # On Python 2 this will be the backported package. @@ -273,6 +278,11 @@ def _error_about_old_config_files(basedir, generated_files): raise e +def ensure_text_and_abspath_expanduser_unicode(basedir): + # type: (Union[bytes, str]) -> str + return abspath_expanduser_unicode(ensure_text(basedir)) + + @attr.s class _Config(object): """ @@ -300,8 +310,8 @@ class _Config(object): config = attr.ib(validator=attr.validators.instance_of(configparser.ConfigParser)) portnum_fname = attr.ib() _basedir = attr.ib( - converter=lambda basedir: abspath_expanduser_unicode(ensure_text(basedir)), - ) + converter=ensure_text_and_abspath_expanduser_unicode, + ) # type: str config_path = attr.ib( validator=attr.validators.optional( attr.validators.instance_of(FilePath), From 3fd46f94009101fdcfff8617ccb335bab255c57f Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 26 Dec 2020 13:02:16 -0500 Subject: [PATCH 73/81] Ignore additional attribute on the function. --- src/allmydata/test/test_node.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/test_node.py b/src/allmydata/test/test_node.py index 1e0f3020c..e44fd5743 100644 --- a/src/allmydata/test/test_node.py +++ b/src/allmydata/test/test_node.py @@ -564,7 +564,7 @@ class TestMissingPorts(unittest.TestCase): config = config_from_string(self.basedir, "portnum", config_data) with self.assertRaises(PortAssignmentRequired): _tub_portlocation(config, None, None) - test_listen_on_zero_with_host.todo = native_str( + test_listen_on_zero_with_host.todo = native_str( # type: ignore "https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3563" ) From cc5a1046d9d936712577717f1ba0409378d279e6 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 26 Dec 2020 13:07:12 -0500 Subject: [PATCH 74/81] Define type for IntroducerService.VERSION, accepting bytes or str as keys for now. --- src/allmydata/introducer/server.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/allmydata/introducer/server.py b/src/allmydata/introducer/server.py index 237c30315..339c5a0ac 100644 --- a/src/allmydata/introducer/server.py +++ b/src/allmydata/introducer/server.py @@ -15,6 +15,12 @@ from past.builtins import long from six import ensure_text import time, os.path, textwrap + +try: + from typing import Any, Dict, Union +except ImportError: + pass + from zope.interface import implementer from twisted.application import service from twisted.internet import defer @@ -147,10 +153,12 @@ class IntroducerService(service.MultiService, Referenceable): name = "introducer" # v1 is the original protocol, added in 1.0 (but only advertised starting # in 1.3), removed in 1.12. v2 is the new signed protocol, added in 1.10 - VERSION = { #"http://allmydata.org/tahoe/protocols/introducer/v1": { }, + # TODO: reconcile bytes/str for keys + VERSION = { + #"http://allmydata.org/tahoe/protocols/introducer/v1": { }, b"http://allmydata.org/tahoe/protocols/introducer/v2": { }, b"application-version": allmydata.__full_version__.encode("utf-8"), - } + } # type: Dict[Union[bytes, str], Any] def __init__(self): service.MultiService.__init__(self) From 854c22e1ca44baa943e36c964c9d06cfa71f8809 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 26 Dec 2020 13:09:57 -0500 Subject: [PATCH 75/81] Use compatible import for urllib.parse.quote. --- src/allmydata/test/web/test_root.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/allmydata/test/web/test_root.py b/src/allmydata/test/web/test_root.py index 0715c8102..2ea418047 100644 --- a/src/allmydata/test/web/test_root.py +++ b/src/allmydata/test/web/test_root.py @@ -1,8 +1,7 @@ import time -from urllib import ( - quote, -) + +from six.moves.urllib.parse import quote from bs4 import ( BeautifulSoup, From 652222116665f6876e476e5a03cf1129fbc1c789 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 26 Dec 2020 13:12:11 -0500 Subject: [PATCH 76/81] Suppress error on SpyHandler interface. --- src/allmydata/test/test_storage_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/test_storage_client.py b/src/allmydata/test/test_storage_client.py index 45a99979f..8500d6bff 100644 --- a/src/allmydata/test/test_storage_client.py +++ b/src/allmydata/test/test_storage_client.py @@ -570,7 +570,7 @@ class SpyEndpoint(object): return d -@implementer(IConnectionHintHandler) +@implementer(IConnectionHintHandler) # type: ignore # warner/foolscap#78 @attr.s class SpyHandler(object): """ From 9780f8bfdcb29120149e4a49254a20fd7576e4ce Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 26 Dec 2020 13:23:28 -0500 Subject: [PATCH 77/81] Add newsfragment --- newsfragments/3399.feature | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/3399.feature diff --git a/newsfragments/3399.feature b/newsfragments/3399.feature new file mode 100644 index 000000000..d30a91679 --- /dev/null +++ b/newsfragments/3399.feature @@ -0,0 +1 @@ +Added 'typechecks' environment for tox running mypy and performing static typechecks. From 4a9d3bde5b53bb7ce9ae38ca03f416f9a4f5aad4 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 26 Dec 2020 13:30:32 -0500 Subject: [PATCH 78/81] Exclude allmydata.scripts.types_ module from PythonTwoRegressions. --- src/allmydata/test/test_python2_regressions.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/allmydata/test/test_python2_regressions.py b/src/allmydata/test/test_python2_regressions.py index 84484f1cf..5c6a654c1 100644 --- a/src/allmydata/test/test_python2_regressions.py +++ b/src/allmydata/test/test_python2_regressions.py @@ -16,6 +16,7 @@ from testtools.matchers import ( BLACKLIST = { "allmydata.test.check_load", "allmydata.windows.registry", + "allmydata.scripts.types_", } From 3951257cd7090188d33f3efabeca263c954912d0 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 27 Dec 2020 16:17:59 -0500 Subject: [PATCH 79/81] Switch to mypy-zope main branch. --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 26e056dc1..706899ebe 100644 --- a/tox.ini +++ b/tox.ini @@ -117,7 +117,7 @@ commands = skip_install = True deps = mypy - git+https://github.com/jaraco/mypy-zope@bugfix/21-InterfaceClass-subclass + git+https://github.com/Shoobx/mypy-zope git+https://github.com/jaraco/foolscap@bugfix/75-use-metaclass commands = mypy src From a01078ddec86e1d193c7940087d6506d376028d8 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 15 Jan 2021 15:00:57 -0500 Subject: [PATCH 80/81] Switch to one of our Docker images for typecheck CI --- .circleci/config.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ff14d6dd3..29b55ad5f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -481,13 +481,14 @@ jobs: typechecks: docker: - <<: *DOCKERHUB_AUTH - image: "jaraco/multipy-tox" + image: "tahoelafsci/ubuntu:18.04-py3" steps: - "checkout" - run: name: "Validate Types" - command: tox -e typechecks + command: | + /tmp/venv/bin/tox -e typechecks build-image: &BUILD_IMAGE # This is a template for a job to build a Docker image that has as much of From c7a4cdb44d801034de1e068f10aa3d8df8a4950f Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 15 Jan 2021 15:25:07 -0500 Subject: [PATCH 81/81] Rely on main branch of foolscap for typechecks. --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 706899ebe..2873bded8 100644 --- a/tox.ini +++ b/tox.ini @@ -118,7 +118,7 @@ skip_install = True deps = mypy git+https://github.com/Shoobx/mypy-zope - git+https://github.com/jaraco/foolscap@bugfix/75-use-metaclass + git+https://github.com/warner/foolscap commands = mypy src