ExternalVendorCode/tahoe-lafs
1
0
Fork 0
mirror of https://github.com/tahoe-lafs/tahoe-lafs.git synced 2025-06-04 08:40:53 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge pull request #845 from tahoe-lafs/3455.python-3-port-node-round1

Browse Source 
3455: Round 1 of porting `allmydata.node`
This commit is contained in:
Ross Patterson 2020-10-05 08:32:28 -07:00 committed by GitHub
commit e89bbe1601
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 84 additions and 75 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
Makefile
View File

@ -51,6 +51,7 @@ test: .tox/create-venvs.log
## Run all tests with coverage collection and reporting. ## Run all tests with coverage collection and reporting.
test-venv-coverage: test-venv-coverage:
# Special handling for reporting coverage even when the test run fails # Special handling for reporting coverage even when the test run fails
rm -f ./.coverage.*
test_exit= test_exit=
$(VIRTUAL_ENV)/bin/coverage run -m twisted.trial --rterrors --reporter=timing \ $(VIRTUAL_ENV)/bin/coverage run -m twisted.trial --rterrors --reporter=timing \
$(TEST_SUITE) || test_exit="$$?" $(TEST_SUITE) || test_exit="$$?"

1
newsfragments/3455.minor Normal file
View File

@ -0,0 +1 @@
Begin porting the `node` module to Python 3.

4
src/allmydata/client.py
View File

@ -69,8 +69,8 @@ def _is_valid_section(section_name):
Currently considers all possible storage server plugin sections valid. Currently considers all possible storage server plugin sections valid.
""" """
return ( return (
section_name.startswith(b"storageserver.plugins.") or section_name.startswith("storageserver.plugins.") or
section_name.startswith(b"storageclient.plugins.") section_name.startswith("storageclient.plugins.")
) )

17
src/allmydata/node.py
View File

@ -9,11 +9,16 @@ import os.path
import re import re
import types import types
import errno import errno
from six.moves import configparser from io import StringIO
import tempfile import tempfile
from io import BytesIO
from base64 import b32decode, b32encode from base64 import b32decode, b32encode
# BBB: Python 2 compatibility
from six.moves import configparser
from future.utils import PY2
if PY2:
from io import BytesIO as StringIO # noqa: F811
from twisted.python import log as twlog from twisted.python import log as twlog
from twisted.application import service from twisted.application import service
from twisted.python.failure import Failure from twisted.python.failure import Failure
@ -70,7 +75,7 @@ def _common_valid_config():
# Add our application versions to the data that Foolscap's LogPublisher # Add our application versions to the data that Foolscap's LogPublisher
# reports. # reports.
for thing, things_version in get_package_versions().items(): for thing, things_version in get_package_versions().items():
app_versions.add_version(thing, str(things_version)) app_versions.add_version(thing, things_version)
# group 1 will be addr (dotted quad string), group 3 if any will be portnum (string) # group 1 will be addr (dotted quad string), group 3 if any will be portnum (string)
ADDR_RE = re.compile("^([1-9][0-9]*\.[1-9][0-9]*\.[1-9][0-9]*\.[1-9][0-9]*)(:([1-9][0-9]*))?$") ADDR_RE = re.compile("^([1-9][0-9]*\.[1-9][0-9]*\.[1-9][0-9]*\.[1-9][0-9]*)(:([1-9][0-9]*))?$")
@ -206,7 +211,7 @@ def config_from_string(basedir, portnumfile, config_str, _valid_config=None):
# load configuration from in-memory string # load configuration from in-memory string
parser = configparser.SafeConfigParser() parser = configparser.SafeConfigParser()
parser.readfp(BytesIO(config_str)) parser.readfp(StringIO(config_str))
fname = "<in-memory>" fname = "<in-memory>"
configutil.validate_config(fname, parser, _valid_config) configutil.validate_config(fname, parser, _valid_config)
@ -821,9 +826,9 @@ class Node(service.MultiService):
for o in twlog.theLogPublisher.observers: for o in twlog.theLogPublisher.observers:
# o might be a FileLogObserver's .emit method # o might be a FileLogObserver's .emit method
if type(o) is type(self.setup_logging): # bound method if type(o) is type(self.setup_logging): # bound method
ob = o.im_self ob = o.__self__
if isinstance(ob, twlog.FileLogObserver): if isinstance(ob, twlog.FileLogObserver):
newmeth = types.UnboundMethodType(formatTimeTahoeStyle, ob, ob.__class__) newmeth = types.MethodType(formatTimeTahoeStyle, ob)
ob.formatTime = newmeth ob.formatTime = newmeth
# TODO: twisted >2.5.0 offers maxRotatedFiles=50 # TODO: twisted >2.5.0 offers maxRotatedFiles=50

10
src/allmydata/test/common.py
View File

@ -94,9 +94,9 @@ from .common_util import ShouldFailMixin # noqa: F401
TEST_RSA_KEY_SIZE = 522 TEST_RSA_KEY_SIZE = 522
EMPTY_CLIENT_CONFIG = config_from_string( EMPTY_CLIENT_CONFIG = config_from_string(
b"/dev/null", "/dev/null",
b"tub.port", "tub.port",
b"" ""
) )
@ -249,8 +249,8 @@ class UseNode(object):
self.config = config_from_string( self.config = config_from_string(
self.basedir.asTextMode().path, self.basedir.asTextMode().path,
u"tub.port", "tub.port",
b""" """
[node] [node]
{node_config} {node_config}

2
src/allmydata/test/eliotutil.py
View File

@ -106,7 +106,7 @@ def eliot_logged_test(f):
# Begin an action that should comprise all messages from the decorated # Begin an action that should comprise all messages from the decorated
# test method. # test method.
with RUN_TEST(name=self.id().decode("utf-8")).context() as action: with RUN_TEST(name=self.id()).context() as action:
# When the test method Deferred fires, the RUN_TEST action is # When the test method Deferred fires, the RUN_TEST action is
# done. However, we won't have re-published the MemoryLogger # done. However, we won't have re-published the MemoryLogger
# messages into the global/default logger when this Deferred # messages into the global/default logger when this Deferred

120
src/allmydata/test/test_client.py
View File

@ -252,11 +252,11 @@ class Basic(testutil.ReallyEqualMixin, unittest.TestCase):
is not set. is not set.
""" """
config = client.config_from_string( config = client.config_from_string(
b"test_storage_default_anonymous_enabled", "test_storage_default_anonymous_enabled",
b"tub.port", "tub.port",
BASECONFIG + ( BASECONFIG + (
b"[storage]\n" "[storage]\n"
b"enabled = true\n" "enabled = true\n"
) )
) )
self.assertTrue(client.anonymous_storage_enabled(config)) self.assertTrue(client.anonymous_storage_enabled(config))
@ -268,11 +268,11 @@ class Basic(testutil.ReallyEqualMixin, unittest.TestCase):
""" """
config = client.config_from_string( config = client.config_from_string(
self.id(), self.id(),
b"tub.port", "tub.port",
BASECONFIG + ( BASECONFIG + (
b"[storage]\n" "[storage]\n"
b"enabled = true\n" "enabled = true\n"
b"anonymous = true\n" "anonymous = true\n"
) )
) )
self.assertTrue(client.anonymous_storage_enabled(config)) self.assertTrue(client.anonymous_storage_enabled(config))
@ -284,11 +284,11 @@ class Basic(testutil.ReallyEqualMixin, unittest.TestCase):
""" """
config = client.config_from_string( config = client.config_from_string(
self.id(), self.id(),
b"tub.port", "tub.port",
BASECONFIG + ( BASECONFIG + (
b"[storage]\n" "[storage]\n"
b"enabled = true\n" "enabled = true\n"
b"anonymous = false\n" "anonymous = false\n"
) )
) )
self.assertFalse(client.anonymous_storage_enabled(config)) self.assertFalse(client.anonymous_storage_enabled(config))
@ -300,11 +300,11 @@ class Basic(testutil.ReallyEqualMixin, unittest.TestCase):
""" """
config = client.config_from_string( config = client.config_from_string(
self.id(), self.id(),
b"tub.port", "tub.port",
BASECONFIG + ( BASECONFIG + (
b"[storage]\n" "[storage]\n"
b"enabled = false\n" "enabled = false\n"
b"anonymous = true\n" "anonymous = true\n"
) )
) )
self.assertFalse(client.anonymous_storage_enabled(config)) self.assertFalse(client.anonymous_storage_enabled(config))
@ -680,11 +680,11 @@ class AnonymousStorage(SyncTestCase):
os.makedirs(basedir + b"/private") os.makedirs(basedir + b"/private")
config = client.config_from_string( config = client.config_from_string(
basedir, basedir,
b"tub.port", "tub.port",
BASECONFIG_I % (SOME_FURL,) + ( BASECONFIG_I % (SOME_FURL,) + (
b"[storage]\n" "[storage]\n"
b"enabled = true\n" "enabled = true\n"
b"anonymous = true\n" "anonymous = true\n"
) )
) )
node = yield client.create_client_from_config( node = yield client.create_client_from_config(
@ -711,11 +711,11 @@ class AnonymousStorage(SyncTestCase):
os.makedirs(basedir + b"/private") os.makedirs(basedir + b"/private")
config = client.config_from_string( config = client.config_from_string(
basedir, basedir,
b"tub.port", "tub.port",
BASECONFIG_I % (SOME_FURL,) + ( BASECONFIG_I % (SOME_FURL,) + (
b"[storage]\n" "[storage]\n"
b"enabled = true\n" "enabled = true\n"
b"anonymous = false\n" "anonymous = false\n"
) )
) )
node = yield client.create_client_from_config( node = yield client.create_client_from_config(
@ -732,7 +732,7 @@ class AnonymousStorage(SyncTestCase):
]), ]),
) )
self.expectThat( self.expectThat(
config.get_private_config(b"storage.furl", default=None), config.get_private_config("storage.furl", default=None),
Is(None), Is(None),
) )
@ -748,18 +748,18 @@ class AnonymousStorage(SyncTestCase):
os.makedirs(basedir + b"/private") os.makedirs(basedir + b"/private")
enabled_config = client.config_from_string( enabled_config = client.config_from_string(
basedir, basedir,
b"tub.port", "tub.port",
BASECONFIG_I % (SOME_FURL,) + ( BASECONFIG_I % (SOME_FURL,) + (
b"[storage]\n" "[storage]\n"
b"enabled = true\n" "enabled = true\n"
b"anonymous = true\n" "anonymous = true\n"
) )
) )
node = yield client.create_client_from_config( node = yield client.create_client_from_config(
enabled_config, enabled_config,
_introducer_factory=MemoryIntroducerClient, _introducer_factory=MemoryIntroducerClient,
) )
anonymous_storage_furl = enabled_config.get_private_config(b"storage.furl") anonymous_storage_furl = enabled_config.get_private_config("storage.furl")
def check_furl(): def check_furl():
return node.tub.getReferenceForURL(anonymous_storage_furl) return node.tub.getReferenceForURL(anonymous_storage_furl)
# Perform a sanity check that our test code makes sense: is this a # Perform a sanity check that our test code makes sense: is this a
@ -772,11 +772,11 @@ class AnonymousStorage(SyncTestCase):
disabled_config = client.config_from_string( disabled_config = client.config_from_string(
basedir, basedir,
b"tub.port", "tub.port",
BASECONFIG_I % (SOME_FURL,) + ( BASECONFIG_I % (SOME_FURL,) + (
b"[storage]\n" "[storage]\n"
b"enabled = true\n" "enabled = true\n"
b"anonymous = false\n" "anonymous = false\n"
) )
) )
node = yield client.create_client_from_config( node = yield client.create_client_from_config(
@ -1137,8 +1137,8 @@ class StorageAnnouncementTests(SyncTestCase):
create_node_dir(self.basedir, u"") create_node_dir(self.basedir, u"")
def get_config(self, storage_enabled, more_storage=b"", more_sections=b""): def get_config(self, storage_enabled, more_storage="", more_sections=""):
return b""" return """
[node] [node]
tub.location = tcp:192.0.2.0:1234 tub.location = tcp:192.0.2.0:1234
@ -1163,7 +1163,7 @@ introducer.furl = pb://abcde@nowhere/fake
""" """
config = client.config_from_string( config = client.config_from_string(
self.basedir, self.basedir,
u"tub.port", "tub.port",
self.get_config(storage_enabled=False), self.get_config(storage_enabled=False),
) )
self.assertThat( self.assertThat(
@ -1185,7 +1185,7 @@ introducer.furl = pb://abcde@nowhere/fake
""" """
config = client.config_from_string( config = client.config_from_string(
self.basedir, self.basedir,
u"tub.port", "tub.port",
self.get_config(storage_enabled=True), self.get_config(storage_enabled=True),
) )
client_deferred = client.create_client_from_config( client_deferred = client.create_client_from_config(
@ -1217,13 +1217,13 @@ introducer.furl = pb://abcde@nowhere/fake
value = u"thing" value = u"thing"
config = client.config_from_string( config = client.config_from_string(
self.basedir, self.basedir,
u"tub.port", "tub.port",
self.get_config( self.get_config(
storage_enabled=True, storage_enabled=True,
more_storage=b"plugins=tahoe-lafs-dummy-v1", more_storage="plugins=tahoe-lafs-dummy-v1",
more_sections=( more_sections=(
b"[storageserver.plugins.tahoe-lafs-dummy-v1]\n" "[storageserver.plugins.tahoe-lafs-dummy-v1]\n"
b"some = {}\n".format(value) "some = {}\n".format(value)
), ),
), ),
) )
@ -1258,15 +1258,15 @@ introducer.furl = pb://abcde@nowhere/fake
config = client.config_from_string( config = client.config_from_string(
self.basedir, self.basedir,
u"tub.port", "tub.port",
self.get_config( self.get_config(
storage_enabled=True, storage_enabled=True,
more_storage=b"plugins=tahoe-lafs-dummy-v1,tahoe-lafs-dummy-v2", more_storage="plugins=tahoe-lafs-dummy-v1,tahoe-lafs-dummy-v2",
more_sections=( more_sections=(
b"[storageserver.plugins.tahoe-lafs-dummy-v1]\n" "[storageserver.plugins.tahoe-lafs-dummy-v1]\n"
b"some = thing-1\n" "some = thing-1\n"
b"[storageserver.plugins.tahoe-lafs-dummy-v2]\n" "[storageserver.plugins.tahoe-lafs-dummy-v2]\n"
b"some = thing-2\n" "some = thing-2\n"
), ),
), ),
) )
@ -1306,13 +1306,13 @@ introducer.furl = pb://abcde@nowhere/fake
config = client.config_from_string( config = client.config_from_string(
self.basedir, self.basedir,
u"tub.port", "tub.port",
self.get_config( self.get_config(
storage_enabled=True, storage_enabled=True,
more_storage=b"plugins=tahoe-lafs-dummy-v1", more_storage="plugins=tahoe-lafs-dummy-v1",
more_sections=( more_sections=(
b"[storageserver.plugins.tahoe-lafs-dummy-v1]\n" "[storageserver.plugins.tahoe-lafs-dummy-v1]\n"
b"some = thing\n" "some = thing\n"
), ),
), ),
) )
@ -1342,10 +1342,10 @@ introducer.furl = pb://abcde@nowhere/fake
config = client.config_from_string( config = client.config_from_string(
self.basedir, self.basedir,
u"tub.port", "tub.port",
self.get_config( self.get_config(
storage_enabled=True, storage_enabled=True,
more_storage=b"plugins=tahoe-lafs-dummy-v1", more_storage="plugins=tahoe-lafs-dummy-v1",
), ),
) )
self.assertThat( self.assertThat(
@ -1380,14 +1380,14 @@ introducer.furl = pb://abcde@nowhere/fake
config = client.config_from_string( config = client.config_from_string(
self.basedir, self.basedir,
u"tub.port", "tub.port",
self.get_config( self.get_config(
storage_enabled=True, storage_enabled=True,
more_storage=b"plugins=tahoe-lafs-dummy-v1", more_storage="plugins=tahoe-lafs-dummy-v1",
more_sections=( more_sections=(
b"[storageserver.plugins.tahoe-lafs-dummy-v1]\n" "[storageserver.plugins.tahoe-lafs-dummy-v1]\n"
# This will make it explode on instantiation. # This will make it explode on instantiation.
b"invalid = configuration\n" "invalid = configuration\n"
) )
), ),
) )
@ -1407,10 +1407,10 @@ introducer.furl = pb://abcde@nowhere/fake
""" """
config = client.config_from_string( config = client.config_from_string(
self.basedir, self.basedir,
u"tub.port", "tub.port",
self.get_config( self.get_config(
storage_enabled=True, storage_enabled=True,
more_storage=b"plugins=tahoe-lafs-dummy-vX", more_storage="plugins=tahoe-lafs-dummy-vX",
), ),
) )
self.assertThat( self.assertThat(

4
src/allmydata/test/test_node.py
View File

@ -520,7 +520,6 @@ introducer.furl = empty
enabled = false enabled = false
[i2p] [i2p]
enabled = false enabled = false
[node]
""" """
NOLISTEN = """ NOLISTEN = """
@ -566,6 +565,7 @@ class Listeners(unittest.TestCase):
create_node_dir(basedir, "testing") create_node_dir(basedir, "testing")
with open(os.path.join(basedir, "tahoe.cfg"), "w") as f: with open(os.path.join(basedir, "tahoe.cfg"), "w") as f:
f.write(BASE_CONFIG) f.write(BASE_CONFIG)
f.write("[node]\n")
f.write("tub.port = tcp:0\n") f.write("tub.port = tcp:0\n")
f.write("tub.location = AUTO\n") f.write("tub.location = AUTO\n")
@ -594,6 +594,7 @@ class Listeners(unittest.TestCase):
location = "tcp:localhost:%d,tcp:localhost:%d" % (port1, port2) location = "tcp:localhost:%d,tcp:localhost:%d" % (port1, port2)
with open(os.path.join(basedir, "tahoe.cfg"), "w") as f: with open(os.path.join(basedir, "tahoe.cfg"), "w") as f:
f.write(BASE_CONFIG) f.write(BASE_CONFIG)
f.write("[node]\n")
f.write("tub.port = %s\n" % port) f.write("tub.port = %s\n" % port)
f.write("tub.location = %s\n" % location) f.write("tub.location = %s\n" % location)
@ -617,6 +618,7 @@ class Listeners(unittest.TestCase):
os.mkdir(os.path.join(basedir, "private")) os.mkdir(os.path.join(basedir, "private"))
with open(config_fname, "w") as f: with open(config_fname, "w") as f:
f.write(BASE_CONFIG) f.write(BASE_CONFIG)
f.write("[node]\n")
f.write("tub.port = listen:i2p,listen:tor\n") f.write("tub.port = listen:i2p,listen:tor\n")
f.write("tub.location = tcp:example.org:1234\n") f.write("tub.location = tcp:example.org:1234\n")
config = client.read_config(basedir, "client.port") config = client.read_config(basedir, "client.port")