ExternalVendorCode/tahoe-lafs
1
0
Fork 0
mirror of https://github.com/tahoe-lafs/tahoe-lafs.git synced 2025-03-10 22:43:52 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge pull request #1277 from tahoe-lafs/3996.newer-flake8-and-pylint

Browse Source 
Fixes: ticket:3996
Fixes: ticket:3997
This commit is contained in:
Jean-Paul Calderone 2023-03-26 16:04:05 -04:00 committed by GitHub
commit b4670028c4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
29 changed files with 140 additions and 381 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
misc/checkers/check_load.py
View File

@ -33,20 +33,11 @@ a mean of 10kB and a max of 100MB, so filesize=min(int(1.0/random(.0002)),1e8)
"""
from __future__ import annotations
import os, sys, httplib, binascii
import urllib, json, random, time, urlparse
try:
from typing import Dict
except ImportError:
pass
# Python 2 compatibility
from future.utils import PY2
if PY2:
from future.builtins import str # noqa: F401
if sys.argv[1] == "--stats":
statsfiles = sys.argv[2:]
# gather stats every 10 seconds, do a moving-window average of the last
@ -54,9 +45,9 @@ if sys.argv[1] == "--stats":
DELAY = 10
MAXSAMPLES = 6
totals = []
last_stats = {} # type: Dict[str, float]
last_stats : dict[str, float] = {}
while True:
stats = {} # type: Dict[str, float]
stats : dict[str, float] = {}
for sf in statsfiles:
for line in open(sf, "r").readlines():
name, str_value = line.split(":")

0
newsfragments/3996.minor Normal file
View File

1
newsfragments/3997.installation Normal file
View File

@ -0,0 +1 @@
Tahoe-LAFS is incompatible with cryptography >= 40 and now declares a requirement on an older version.

8
setup.py
View File

@ -63,7 +63,11 @@ install_requires = [
# Twisted[conch] also depends on cryptography and Twisted[tls]
# transitively depends on cryptography. So it's anyone's guess what
# version of cryptography will *really* be installed.
"cryptography >= 2.6",
# * cryptography 40 broke constants we need; should really be using them
# * via pyOpenSSL; will be fixed in
# * https://github.com/pyca/pyopenssl/issues/1201
"cryptography >= 2.6, < 40",
# * The SFTP frontend depends on Twisted 11.0.0 to fix the SSH server
# rekeying bug <https://twistedmatrix.com/trac/ticket/4395>
@ -400,7 +404,7 @@ setup(name="tahoe-lafs", # also set in __init__.py
# disagreeing on what is or is not a lint issue. We can bump
# this version from time to time, but we will do it
# intentionally.
"pyflakes == 2.2.0",
"pyflakes == 3.0.1",
"coverage ~= 5.0",
"mock",
"tox ~= 3.0",

20
src/allmydata/immutable/upload.py
View File

@ -2,22 +2,12 @@
Ported to Python 3.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
from __future__ import annotations
from future.utils import PY2, native_str
if PY2:
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
from future.utils import native_str
from past.builtins import long, unicode
from six import ensure_str
try:
from typing import List
except ImportError:
pass
import os, time, weakref, itertools
import attr
@ -915,12 +905,12 @@ class _Accum(object):
:ivar remaining: The number of bytes still expected.
:ivar ciphertext: The bytes accumulated so far.
"""
remaining = attr.ib(validator=attr.validators.instance_of(int)) # type: int
ciphertext = attr.ib(default=attr.Factory(list)) # type: List[bytes]
remaining : int = attr.ib(validator=attr.validators.instance_of(int))
ciphertext : list[bytes] = attr.ib(default=attr.Factory(list))
def extend(self,
size, # type: int
ciphertext, # type: List[bytes]
ciphertext, # type: list[bytes]
):
"""
Accumulate some more ciphertext.

19
src/allmydata/introducer/server.py
View File

@ -2,24 +2,13 @@
Ported to Python 3.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
from __future__ import annotations
from future.utils import PY2
if PY2:
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
from past.builtins import long
from six import ensure_text
import time, os.path, textwrap
try:
from typing import Any, Dict, Union
except ImportError:
pass
from typing import Any, Union
from zope.interface import implementer
from twisted.application import service
@ -161,11 +150,11 @@ class IntroducerService(service.MultiService, Referenceable):
# v1 is the original protocol, added in 1.0 (but only advertised starting
# in 1.3), removed in 1.12. v2 is the new signed protocol, added in 1.10
# TODO: reconcile bytes/str for keys
VERSION = {
VERSION : dict[Union[bytes, str], Any]= {
#"http://allmydata.org/tahoe/protocols/introducer/v1": { },
b"http://allmydata.org/tahoe/protocols/introducer/v2": { },
b"application-version": allmydata.__full_version__.encode("utf-8"),
} # type: Dict[Union[bytes, str], Any]
}
def __init__(self):
service.MultiService.__init__(self)

17
src/allmydata/node.py
View File

@ -4,14 +4,8 @@ a node for Tahoe-LAFS.
Ported to Python 3.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
from __future__ import annotations
from future.utils import PY2
if PY2:
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
from six import ensure_str, ensure_text
import json
@ -23,11 +17,7 @@ import errno
from base64 import b32decode, b32encode
from errno import ENOENT, EPERM
from warnings import warn
try:
from typing import Union
except ImportError:
pass
from typing import Union
import attr
@ -281,8 +271,7 @@ def _error_about_old_config_files(basedir, generated_files):
raise e
def ensure_text_and_abspath_expanduser_unicode(basedir):
# type: (Union[bytes, str]) -> str
def ensure_text_and_abspath_expanduser_unicode(basedir: Union[bytes, str]) -> str:
return abspath_expanduser_unicode(ensure_text(basedir))

4
src/allmydata/scripts/admin.py
View File

@ -255,9 +255,9 @@ def do_admin(options):
return f(so)
subCommands = [
subCommands : SubCommands = [
("admin", None, AdminCommand, "admin subcommands: use 'tahoe admin' for a list"),
] # type: SubCommands
]
dispatch = {
"admin": do_admin,

22
src/allmydata/scripts/cli.py
View File

@ -1,22 +1,10 @@
"""
Ported to Python 3.
"""
from __future__ import unicode_literals
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from future.utils import PY2
if PY2:
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
import os.path, re, fnmatch
try:
from allmydata.scripts.types_ import SubCommands, Parameters
except ImportError:
pass
from allmydata.scripts.types_ import SubCommands, Parameters
from twisted.python import usage
from allmydata.scripts.common import get_aliases, get_default_nodedir, \
@ -29,14 +17,14 @@ NODEURL_RE=re.compile("http(s?)://([^:]*)(:([1-9][0-9]*))?")
_default_nodedir = get_default_nodedir()
class FileStoreOptions(BaseOptions):
optParameters = [
optParameters : Parameters = [
["node-url", "u", None,
"Specify the URL of the Tahoe gateway node, such as "
"'http://127.0.0.1:3456'. "
"This overrides the URL found in the --node-directory ."],
["dir-cap", None, None,
"Specify which dirnode URI should be used as the 'tahoe' alias."]
] # type: Parameters
]
def postOptions(self):
self["quiet"] = self.parent["quiet"]
@ -484,7 +472,7 @@ class DeepCheckOptions(FileStoreOptions):
(which must be a directory), like 'tahoe check' but for multiple files.
Optionally repair any problems found."""
subCommands = [
subCommands : SubCommands = [
("mkdir", None, MakeDirectoryOptions, "Create a new directory."),
("add-alias", None, AddAliasOptions, "Add a new alias cap."),
("create-alias", None, CreateAliasOptions, "Create a new alias cap."),
@ -503,7 +491,7 @@ subCommands = [
("check", None, CheckOptions, "Check a single file or directory."),
("deep-check", None, DeepCheckOptions, "Check all files/directories reachable from a starting point."),
("status", None, TahoeStatusCommand, "Various status information."),
] # type: SubCommands
]
def mkdir(options):
from allmydata.scripts import tahoe_mkdir

36
src/allmydata/scripts/common.py
View File

@ -4,29 +4,13 @@
Ported to Python 3.
"""
from __future__ import unicode_literals
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from future.utils import PY2
if PY2:
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
else:
from typing import Union
from typing import Union, Optional
import os, sys, textwrap
import codecs
from os.path import join
import urllib.parse
try:
from typing import Optional
from .types_ import Parameters
except ImportError:
pass
from yaml import (
safe_dump,
)
@ -37,6 +21,8 @@ from allmydata.util.assertutil import precondition
from allmydata.util.encodingutil import quote_output, \
quote_local_unicode_path, argv_to_abspath
from allmydata.scripts.default_nodedir import _default_nodedir
from .types_ import Parameters
def get_default_nodedir():
return _default_nodedir
@ -59,7 +45,7 @@ class BaseOptions(usage.Options):
def opt_version(self):
raise usage.UsageError("--version not allowed on subcommands")
description = None # type: Optional[str]
description : Optional[str] = None
description_unwrapped = None # type: Optional[str]
def __str__(self):
@ -80,10 +66,10 @@ class BaseOptions(usage.Options):
class BasedirOptions(BaseOptions):
default_nodedir = _default_nodedir
optParameters = [
optParameters : Parameters = [
["basedir", "C", None, "Specify which Tahoe base directory should be used. [default: %s]"
% quote_local_unicode_path(_default_nodedir)],
] # type: Parameters
]
def parseArgs(self, basedir=None):
# This finds the node-directory option correctly even if we are in a subcommand.
@ -283,9 +269,8 @@ def get_alias(aliases, path_unicode, default):
quote_output(alias))
return uri.from_string_dirnode(aliases[alias]).to_string(), path[colon+1:]
def escape_path(path):
# type: (Union[str,bytes]) -> str
u"""
def escape_path(path: Union[str, bytes]) -> str:
"""
Return path quoted to US-ASCII, valid URL characters.
>>> path = u'/føö/bar/☃'
@ -302,9 +287,4 @@ def escape_path(path):
]),
"ascii"
)
# Eventually (i.e. as part of Python 3 port) we want this to always return
# Unicode strings. However, to reduce diff sizes in the short term it'll
# return native string (i.e. bytes) on Python 2.
if PY2:
result = result.encode("ascii").__native__()
return result

48
src/allmydata/scripts/create_node.py
View File

@ -1,25 +1,11 @@
# Ported to Python 3
from __future__ import print_function
from __future__ import absolute_import
from __future__ import division
from __future__ import unicode_literals
from future.utils import PY2
if PY2:
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
import io
import os
try:
from allmydata.scripts.types_ import (
SubCommands,
Parameters,
Flags,
)
except ImportError:
pass
from allmydata.scripts.types_ import (
SubCommands,
Parameters,
Flags,
)
from twisted.internet import reactor, defer
from twisted.python.usage import UsageError
@ -48,7 +34,7 @@ def write_tac(basedir, nodetype):
fileutil.write(os.path.join(basedir, "tahoe-%s.tac" % (nodetype,)), dummy_tac)
WHERE_OPTS = [
WHERE_OPTS : Parameters = [
("location", None, None,
"Server location to advertise (e.g. tcp:example.org:12345)"),
("port", None, None,
@ -57,29 +43,29 @@ WHERE_OPTS = [
"Hostname to automatically set --location/--port when --listen=tcp"),
("listen", None, "tcp",
"Comma-separated list of listener types (tcp,tor,i2p,none)."),
] # type: Parameters
]
TOR_OPTS = [
TOR_OPTS : Parameters = [
("tor-control-port", None, None,
"Tor's control port endpoint descriptor string (e.g. tcp:127.0.0.1:9051 or unix:/var/run/tor/control)"),
("tor-executable", None, None,
"The 'tor' executable to run (default is to search $PATH)."),
] # type: Parameters
]
TOR_FLAGS = [
TOR_FLAGS : Flags = [
("tor-launch", None, "Launch a tor instead of connecting to a tor control port."),
] # type: Flags
]
I2P_OPTS = [
I2P_OPTS : Parameters = [
("i2p-sam-port", None, None,
"I2P's SAM API port endpoint descriptor string (e.g. tcp:127.0.0.1:7656)"),
("i2p-executable", None, None,
"(future) The 'i2prouter' executable to run (default is to search $PATH)."),
] # type: Parameters
]
I2P_FLAGS = [
I2P_FLAGS : Flags = [
("i2p-launch", None, "(future) Launch an I2P router instead of connecting to a SAM API port."),
] # type: Flags
]
def validate_where_options(o):
if o['listen'] == "none":
@ -508,11 +494,11 @@ def create_introducer(config):
defer.returnValue(0)
subCommands = [
subCommands : SubCommands = [
("create-node", None, CreateNodeOptions, "Create a node that acts as a client, server or both."),
("create-client", None, CreateClientOptions, "Create a client node (with storage initially disabled)."),
("create-introducer", None, CreateIntroducerOptions, "Create an introducer node."),
] # type: SubCommands
]
dispatch = {
"create-node": create_node,

18
src/allmydata/scripts/debug.py
View File

@ -1,19 +1,8 @@
"""
Ported to Python 3.
"""
from __future__ import unicode_literals
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from future.utils import PY2, bchr
if PY2:
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
try:
from allmydata.scripts.types_ import SubCommands
except ImportError:
pass
from future.utils import bchr
import struct, time, os, sys
@ -31,6 +20,7 @@ from allmydata.mutable.common import NeedMoreDataError
from allmydata.immutable.layout import ReadBucketProxy
from allmydata.util import base32
from allmydata.util.encodingutil import quote_output
from allmydata.scripts.types_ import SubCommands
class DumpOptions(BaseOptions):
def getSynopsis(self):
@ -1076,9 +1066,9 @@ def do_debug(options):
return f(so)
subCommands = [
subCommands : SubCommands = [
("debug", None, DebugCommand, "debug subcommands: use 'tahoe debug' for a list."),
] # type: SubCommands
]
dispatch = {
"debug": do_debug,

21
src/allmydata/scripts/runner.py
View File

@ -1,28 +1,15 @@
from __future__ import print_function
from __future__ import absolute_import
from __future__ import division
from __future__ import unicode_literals
from future.utils import PY2
if PY2:
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
import os, sys
from six.moves import StringIO
from io import StringIO
from past.builtins import unicode
import six
try:
from allmydata.scripts.types_ import SubCommands
except ImportError:
pass
from twisted.python import usage
from twisted.internet import defer, task, threads
from allmydata.scripts.common import get_default_nodedir
from allmydata.scripts import debug, create_node, cli, \
admin, tahoe_run, tahoe_invite
from allmydata.scripts.types_ import SubCommands
from allmydata.util.encodingutil import quote_local_unicode_path, argv_to_unicode
from allmydata.util.eliotutil import (
opt_eliot_destination,
@ -47,9 +34,9 @@ if _default_nodedir:
NODEDIR_HELP += " [default for most commands: " + quote_local_unicode_path(_default_nodedir) + "]"
process_control_commands = [
process_control_commands : SubCommands = [
("run", None, tahoe_run.RunOptions, "run a node without daemonizing"),
] # type: SubCommands
]
class Options(usage.Options):

18
src/allmydata/scripts/tahoe_invite.py
View File

@ -1,19 +1,6 @@
"""
Ported to Python 3.
"""
from __future__ import unicode_literals
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from future.utils import PY2
if PY2:
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
try:
from allmydata.scripts.types_ import SubCommands
except ImportError:
pass
from twisted.python import usage
from twisted.internet import defer, reactor
@ -21,6 +8,7 @@ from twisted.internet import defer, reactor
from allmydata.util.encodingutil import argv_to_abspath
from allmydata.util import jsonbytes as json
from allmydata.scripts.common import get_default_nodedir, get_introducer_furl
from allmydata.scripts.types_ import SubCommands
from allmydata.client import read_config
@ -112,10 +100,10 @@ def invite(options):
print("Completed successfully", file=out)
subCommands = [
subCommands : SubCommands = [
("invite", None, InviteOptions,
"Invite a new node to this grid"),
] # type: SubCommands
]
dispatch = {
"invite": invite,

14
src/allmydata/storage/http_server.py
View File

@ -4,7 +4,7 @@ HTTP server for storage.
from __future__ import annotations
from typing import Dict, List, Set, Tuple, Any, Callable, Union, cast
from typing import Any, Callable, Union, cast
from functools import wraps
from base64 import b64decode
import binascii
@ -67,8 +67,8 @@ class ClientSecretsException(Exception):
def _extract_secrets(
header_values, required_secrets
): # type: (List[str], Set[Secrets]) -> Dict[Secrets, bytes]
header_values: list[str], required_secrets: set[Secrets]
) -> dict[Secrets, bytes]:
"""
Given list of values of ``X-Tahoe-Authorization`` headers, and required
secrets, return dictionary mapping secrets to decoded values.
@ -173,7 +173,7 @@ class UploadsInProgress(object):
_uploads: dict[bytes, StorageIndexUploads] = Factory(dict)
# Map BucketWriter to (storage index, share number)
_bucketwriters: dict[BucketWriter, Tuple[bytes, int]] = Factory(dict)
_bucketwriters: dict[BucketWriter, tuple[bytes, int]] = Factory(dict)
def add_write_bucket(
self,
@ -798,7 +798,9 @@ class HTTPServer(object):
# The reason can be a string with explanation, so in theory it could be
# longish?
info = await self._read_encoded(
request, _SCHEMAS["advise_corrupt_share"], max_size=32768,
request,
_SCHEMAS["advise_corrupt_share"],
max_size=32768,
)
bucket.advise_corrupt_share(info["reason"].encode("utf-8"))
return b""
@ -973,7 +975,7 @@ def listen_tls(
endpoint: IStreamServerEndpoint,
private_key_path: FilePath,
cert_path: FilePath,
) -> Deferred[Tuple[DecodedURL, IListeningPort]]:
) -> Deferred[tuple[DecodedURL, IListeningPort]]:
"""
Start a HTTPS storage server on the given port, return the NURL and the
listening port.

17
src/allmydata/storage/lease_schema.py
View File

@ -2,19 +2,7 @@
Ported to Python 3.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
from future.utils import PY2
if PY2:
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
try:
from typing import Union
except ImportError:
pass
from typing import Union
import attr
@ -95,8 +83,7 @@ class HashedLeaseSerializer(object):
cls._hash_secret,
)
def serialize(self, lease):
# type: (Union[LeaseInfo, HashedLeaseInfo]) -> bytes
def serialize(self, lease: Union[LeaseInfo, HashedLeaseInfo]) -> bytes:
if isinstance(lease, LeaseInfo):
# v2 of the immutable schema stores lease secrets hashed. If
# we're given a LeaseInfo then it holds plaintext secrets. Hash

5
src/allmydata/storage/server.py
View File

@ -2,8 +2,9 @@
Ported to Python 3.
"""
from __future__ import annotations
from future.utils import bytes_to_native_str
from typing import Dict, Tuple, Iterable
from typing import Iterable, Any
import os, re
@ -823,7 +824,7 @@ class FoolscapStorageServer(Referenceable): # type: ignore # warner/foolscap#78
self._server = storage_server
# Canaries and disconnect markers for BucketWriters created via Foolscap:
self._bucket_writer_disconnect_markers = {} # type: Dict[BucketWriter,Tuple[IRemoteReference, object]]
self._bucket_writer_disconnect_markers : dict[BucketWriter, tuple[IRemoteReference, Any]] = {}
self._server.register_bucket_writer_close_handler(self._bucket_writer_closed)

17
src/allmydata/test/cli/test_create.py
View File

@ -1,21 +1,11 @@
"""
Ported to Python 3.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
from future.utils import PY2
if PY2:
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
from __future__ import annotations
import os
try:
from typing import Any, List, Tuple
except ImportError:
pass
from typing import Any
from twisted.trial import unittest
from twisted.internet import defer, reactor
@ -356,8 +346,7 @@ class Config(unittest.TestCase):
self.assertIn("is not empty", err)
self.assertIn("To avoid clobbering anything, I am going to quit now", err)
def fake_config(testcase, module, result):
# type: (unittest.TestCase, Any, Any) -> List[Tuple]
def fake_config(testcase: unittest.TestCase, module: Any, result: Any) -> list[tuple]:
"""
Monkey-patch a fake configuration function into the given module.

22
src/allmydata/test/eliotutil.py
View File

@ -3,18 +3,6 @@ Tools aimed at the interaction between tests and Eliot.
Ported to Python 3.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
# Python 2 compatibility
# Can't use `builtins.str` because it's not JSON encodable:
# `exceptions.TypeError: <class 'future.types.newstr.newstr'> is not JSON-encodeable`
from past.builtins import unicode as str
from future.utils import PY2
if PY2:
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, max, min # noqa: F401
from six import ensure_text
@ -23,11 +11,7 @@ __all__ = [
"EliotLoggedRunTest",
]
try:
from typing import Callable
except ImportError:
pass
from typing import Callable
from functools import (
partial,
wraps,
@ -147,8 +131,8 @@ class EliotLoggedRunTest(object):
def with_logging(
test_id, # type: str
test_method, # type: Callable
test_id: str,
test_method: Callable,
):
"""
Decorate a test method with additional log-related behaviors.

47
src/allmydata/test/no_network.py
View File

@ -1,34 +1,23 @@
"""
Ported to Python 3.
This contains a test harness that creates a full Tahoe grid in a single
process (actually in a single MultiService) which does not use the network.
It does not use an Introducer, and there are no foolscap Tubs. Each storage
server puts real shares on disk, but is accessed through loopback
RemoteReferences instead of over serialized SSL. It is not as complete as
the common.SystemTestMixin framework (which does use the network), but
should be considerably faster: on my laptop, it takes 50-80ms to start up,
whereas SystemTestMixin takes close to 2s.
This should be useful for tests which want to examine and/or manipulate the
uploaded shares, checker/verifier/repairer tests, etc. The clients have no
Tubs, so it is not useful for tests that involve a Helper.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
# This contains a test harness that creates a full Tahoe grid in a single
# process (actually in a single MultiService) which does not use the network.
# It does not use an Introducer, and there are no foolscap Tubs. Each storage
# server puts real shares on disk, but is accessed through loopback
# RemoteReferences instead of over serialized SSL. It is not as complete as
# the common.SystemTestMixin framework (which does use the network), but
# should be considerably faster: on my laptop, it takes 50-80ms to start up,
# whereas SystemTestMixin takes close to 2s.
from __future__ import annotations
# This should be useful for tests which want to examine and/or manipulate the
# uploaded shares, checker/verifier/repairer tests, etc. The clients have no
# Tubs, so it is not useful for tests that involve a Helper.
from future.utils import PY2
if PY2:
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
from past.builtins import unicode
from six import ensure_text
try:
from typing import Dict, Callable
except ImportError:
pass
from typing import Callable
import os
from base64 import b32encode
@ -251,7 +240,7 @@ def create_no_network_client(basedir):
:return: a Deferred yielding an instance of _Client subclass which
does no actual networking but has the same API.
"""
basedir = abspath_expanduser_unicode(unicode(basedir))
basedir = abspath_expanduser_unicode(str(basedir))
fileutil.make_dirs(os.path.join(basedir, "private"), 0o700)
from allmydata.client import read_config
@ -577,8 +566,7 @@ class GridTestMixin(object):
pass
return sorted(shares)
def copy_shares(self, uri):
# type: (bytes) -> Dict[bytes, bytes]
def copy_shares(self, uri: bytes) -> dict[bytes, bytes]:
"""
Read all of the share files for the given capability from the storage area
of the storage servers created by ``set_up_grid``.
@ -630,8 +618,7 @@ class GridTestMixin(object):
with open(i_sharefile, "wb") as f:
f.write(corruptdata)
def corrupt_all_shares(self, uri, corruptor, debug=False):
# type: (bytes, Callable[[bytes, bool], bytes], bool) -> None
def corrupt_all_shares(self, uri: bytes, corruptor: Callable[[bytes, bool], bytes], debug: bool=False):
"""
Apply ``corruptor`` to the contents of all share files associated with a
given capability and replace the share file contents with its result.

16
src/allmydata/test/test_download.py
View File

@ -1,23 +1,14 @@
"""
Ported to Python 3.
"""
from __future__ import print_function
from __future__ import absolute_import
from __future__ import division
from __future__ import unicode_literals
from future.utils import PY2, bchr
if PY2:
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
from future.utils import bchr
# system-level upload+download roundtrip test, but using shares created from
# a previous run. This asserts that the current code is capable of decoding
# shares from a previous version.
try:
from typing import Any
except ImportError:
pass
from typing import Any
import six
import os
@ -1197,8 +1188,7 @@ class Corruption(_Base, unittest.TestCase):
return d
def _corrupt_flip_all(self, ign, imm_uri, which):
# type: (Any, bytes, int) -> None
def _corrupt_flip_all(self, ign: Any, imm_uri: bytes, which: int) -> None:
"""
Flip the least significant bit at a given byte position in all share files
for the given capability.

19
src/allmydata/test/test_helper.py
View File

@ -1,14 +1,7 @@
"""
Ported to Python 3.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
from future.utils import PY2
if PY2:
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
from __future__ import annotations
import os
from struct import (
@ -17,13 +10,8 @@ from struct import (
from functools import (
partial,
)
import attr
try:
from typing import List
from allmydata.introducer.client import IntroducerClient
except ImportError:
pass
import attr
from twisted.internet import defer
from twisted.trial import unittest
@ -35,6 +23,7 @@ from eliot.twisted import (
inline_callbacks,
)
from allmydata.introducer.client import IntroducerClient
from allmydata.crypto import aes
from allmydata.storage.server import (
si_b2a,
@ -132,7 +121,7 @@ class FakeCHKCheckerAndUEBFetcher(object):
))
class FakeClient(service.MultiService):
introducer_clients = [] # type: List[IntroducerClient]
introducer_clients : list[IntroducerClient] = []
DEFAULT_ENCODING_PARAMETERS = {"k":25,
"happy": 75,
"n": 100,

6
src/allmydata/test/test_istorageserver.py
View File

@ -8,9 +8,9 @@ reused across tests, so each test should be careful to generate unique storage
indexes.
"""
from future.utils import bchr
from __future__ import annotations
from typing import Set
from future.utils import bchr
from random import Random
from unittest import SkipTest
@ -1041,7 +1041,7 @@ class IStorageServerMutableAPIsTestsMixin(object):
class _SharedMixin(SystemTestMixin):
"""Base class for Foolscap and HTTP mixins."""
SKIP_TESTS = set() # type: Set[str]
SKIP_TESTS : set[str] = set()
def _get_istorage_server(self):
native_server = next(iter(self.clients[0].storage_broker.get_known_servers()))

19
src/allmydata/uri.py
View File

@ -6,26 +6,11 @@ Ported to Python 3.
Methods ending in to_string() are actually to_bytes(), possibly should be fixed
in follow-up port.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
from future.utils import PY2
if PY2:
# Don't import bytes or str, to prevent future's newbytes leaking and
# breaking code that only expects normal bytes.
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, dict, list, object, range, max, min # noqa: F401
from past.builtins import unicode as str
from past.builtins import unicode, long
import re
try:
from typing import Type
except ImportError:
pass
from typing import Type
from zope.interface import implementer
from twisted.python.components import registerAdapter
@ -707,7 +692,7 @@ class DirectoryURIVerifier(_DirectoryBaseURI):
BASE_STRING=b'URI:DIR2-Verifier:'
BASE_STRING_RE=re.compile(b'^'+BASE_STRING)
INNER_URI_CLASS=SSKVerifierURI # type: Type[IVerifierURI]
INNER_URI_CLASS : Type[IVerifierURI] = SSKVerifierURI
def __init__(self, filenode_uri=None):
if filenode_uri:

60
src/allmydata/util/base32.py
View File

@ -3,30 +3,11 @@ Base32 encoding.
Ported to Python 3.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
from future.utils import PY2
if PY2:
from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
if PY2:
def backwardscompat_bytes(b):
"""
Replace Future bytes with native Python 2 bytes, so % works
consistently until other modules are ported.
"""
return getattr(b, "__native__", lambda: b)()
import string
maketrans = string.maketrans
else:
def backwardscompat_bytes(b):
return b
maketrans = bytes.maketrans
from typing import Optional
def backwardscompat_bytes(b):
return b
maketrans = bytes.maketrans
from typing import Optional
import base64
from allmydata.util.assertutil import precondition
@ -34,7 +15,7 @@ from allmydata.util.assertutil import precondition
rfc3548_alphabet = b"abcdefghijklmnopqrstuvwxyz234567" # RFC3548 standard used by Gnutella, Content-Addressable Web, THEX, Bitzi, Web-Calculus...
chars = rfc3548_alphabet
vals = backwardscompat_bytes(bytes(range(32)))
vals = bytes(range(32))
c2vtranstable = maketrans(chars, vals)
v2ctranstable = maketrans(vals, chars)
identitytranstable = maketrans(b'', b'')
@ -61,16 +42,16 @@ def get_trailing_chars_without_lsbs(N):
d = {}
return b''.join(_get_trailing_chars_without_lsbs(N, d=d))
BASE32CHAR = backwardscompat_bytes(b'['+get_trailing_chars_without_lsbs(0)+b']')
BASE32CHAR_4bits = backwardscompat_bytes(b'['+get_trailing_chars_without_lsbs(1)+b']')
BASE32CHAR_3bits = backwardscompat_bytes(b'['+get_trailing_chars_without_lsbs(2)+b']')
BASE32CHAR_2bits = backwardscompat_bytes(b'['+get_trailing_chars_without_lsbs(3)+b']')
BASE32CHAR_1bits = backwardscompat_bytes(b'['+get_trailing_chars_without_lsbs(4)+b']')
BASE32STR_1byte = backwardscompat_bytes(BASE32CHAR+BASE32CHAR_3bits)
BASE32STR_2bytes = backwardscompat_bytes(BASE32CHAR+b'{3}'+BASE32CHAR_1bits)
BASE32STR_3bytes = backwardscompat_bytes(BASE32CHAR+b'{4}'+BASE32CHAR_4bits)
BASE32STR_4bytes = backwardscompat_bytes(BASE32CHAR+b'{6}'+BASE32CHAR_2bits)
BASE32STR_anybytes = backwardscompat_bytes(bytes(b'((?:%s{8})*') % (BASE32CHAR,) + bytes(b"(?:|%s|%s|%s|%s))") % (BASE32STR_1byte, BASE32STR_2bytes, BASE32STR_3bytes, BASE32STR_4bytes))
BASE32CHAR = b'['+get_trailing_chars_without_lsbs(0)+b']'
BASE32CHAR_4bits = b'['+get_trailing_chars_without_lsbs(1)+b']'
BASE32CHAR_3bits = b'['+get_trailing_chars_without_lsbs(2)+b']'
BASE32CHAR_2bits = b'['+get_trailing_chars_without_lsbs(3)+b']'
BASE32CHAR_1bits = b'['+get_trailing_chars_without_lsbs(4)+b']'
BASE32STR_1byte = BASE32CHAR+BASE32CHAR_3bits
BASE32STR_2bytes = BASE32CHAR+b'{3}'+BASE32CHAR_1bits
BASE32STR_3bytes = BASE32CHAR+b'{4}'+BASE32CHAR_4bits
BASE32STR_4bytes = BASE32CHAR+b'{6}'+BASE32CHAR_2bits
BASE32STR_anybytes = bytes(b'((?:%s{8})*') % (BASE32CHAR,) + bytes(b"(?:|%s|%s|%s|%s))") % (BASE32STR_1byte, BASE32STR_2bytes, BASE32STR_3bytes, BASE32STR_4bytes)
def b2a(os): # type: (bytes) -> bytes
"""
@ -80,7 +61,7 @@ def b2a(os): # type: (bytes) -> bytes
"""
return base64.b32encode(os).rstrip(b"=").lower()
def b2a_or_none(os): # type: (Optional[bytes]) -> Optional[bytes]
def b2a_or_none(os: Optional[bytes]) -> Optional[bytes]:
if os is not None:
return b2a(os)
return None
@ -100,8 +81,6 @@ NUM_OS_TO_NUM_QS=(0, 2, 4, 5, 7,)
NUM_QS_TO_NUM_OS=(0, 1, 1, 2, 2, 3, 3, 4)
NUM_QS_LEGIT=(1, 0, 1, 0, 1, 1, 0, 1,)
NUM_QS_TO_NUM_BITS=tuple([_x*8 for _x in NUM_QS_TO_NUM_OS])
if PY2:
del _x
# A fast way to determine whether a given string *could* be base-32 encoded data, assuming that the
# original data had 8K bits for a positive integer K.
@ -135,8 +114,6 @@ def a2b(cs): # type: (bytes) -> bytes
"""
@param cs the base-32 encoded data (as bytes)
"""
# Workaround Future newbytes issues by converting to real bytes on Python 2:
cs = backwardscompat_bytes(cs)
precondition(could_be_base32_encoded(cs), "cs is required to be possibly base32 encoded data.", cs=cs)
precondition(isinstance(cs, bytes), cs)
@ -144,9 +121,8 @@ def a2b(cs): # type: (bytes) -> bytes
# Add padding back, to make Python's base64 module happy:
while (len(cs) * 5) % 8 != 0:
cs += b"="
# Let newbytes come through and still work on Python 2, where the base64
# module gets confused by them.
return base64.b32decode(backwardscompat_bytes(cs))
return base64.b32decode(cs)
__all__ = ["b2a", "a2b", "b2a_or_none", "BASE32CHAR_3bits", "BASE32CHAR_1bits", "BASE32CHAR", "BASE32STR_anybytes", "could_be_base32_encoded"]

7
src/allmydata/util/deferredutil.py
View File

@ -208,10 +208,9 @@ class WaitForDelayedCallsMixin(PollMixin):
@inline_callbacks
def until(
action, # type: Callable[[], defer.Deferred[Any]]
condition, # type: Callable[[], bool]
):
# type: (...) -> defer.Deferred[None]
action: Callable[[], defer.Deferred[Any]],
condition: Callable[[], bool],
) -> defer.Deferred[None]:
"""
Run a Deferred-returning function until a condition is true.

16
src/allmydata/util/pollmixin.py
View File

@ -4,22 +4,10 @@ Polling utility that returns Deferred.
Ported to Python 3.
"""
from __future__ import print_function
from __future__ import absolute_import
from __future__ import division
from __future__ import unicode_literals
from future.utils import PY2
if PY2:
from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
from __future__ import annotations
import time
try:
from typing import List
except ImportError:
pass
from twisted.internet import task
class TimeoutError(Exception):
@ -29,7 +17,7 @@ class PollComplete(Exception):
pass
class PollMixin(object):
_poll_should_ignore_these_errors = [] # type: List[Exception]
_poll_should_ignore_these_errors : list[Exception] = []
def poll(self, check_f, pollinterval=0.01, timeout=1000):
# Return a Deferred, then call check_f periodically until it returns

2
src/allmydata/web/common.py
View File

@ -117,7 +117,7 @@ def boolean_of_arg(arg): # type: (bytes) -> bool
return arg.lower() in (b"true", b"t", b"1", b"on")
def parse_replace_arg(replace): # type: (bytes) -> Union[bool,_OnlyFiles]
def parse_replace_arg(replace: bytes) -> Union[bool,_OnlyFiles]:
assert isinstance(replace, bytes)
if replace.lower() == b"only-files":
return ONLY_FILES

7
tox.ini
View File

@ -100,10 +100,9 @@ commands =
[testenv:codechecks]
basepython = python3
deps =
# Newer versions of PyLint have buggy configuration
# (https://github.com/PyCQA/pylint/issues/4574), so stick to old version
# for now.
pylint < 2.5
# Make sure we get a version of PyLint that respects config, and isn't too
# old.
pylint < 2.18, >2.14
# On macOS, git inside of towncrier needs $HOME.
passenv = HOME
setenv =