ExternalVendorCode/tahoe-lafs
1
0
Fork 0
mirror of https://github.com/tahoe-lafs/tahoe-lafs.git synced 2024-12-24 15:16:41 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge pull request #772 from tahoe-lafs/3377.configutil-connection_status-python-3

Browse Source 
Port configutil and connection_status to Python 3

Fixes ticket:3377
This commit is contained in:
Itamar Turner-Trauring 2020-08-27 14:51:26 -04:00 committed by GitHub
commit 7da73c1a03
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 245 additions and 181 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

0
newsfragments/3377.minor Normal file
View File

1
src/allmydata/interfaces.py
View File

@ -1,4 +1,3 @@
from past.builtins import long
from zope.interface import Interface, Attribute

27
src/allmydata/test/cli/test_create.py
View File

@ -6,6 +6,8 @@ from twisted.python import usage
from allmydata.util import configutil
from ..common_util import run_cli, parse_cli
from ...scripts import create_node
from ... import client
def read_config(basedir):
tahoe_cfg = os.path.join(basedir, "tahoe.cfg")
@ -33,6 +35,31 @@ class Config(unittest.TestCase):
e = self.assertRaises(usage.UsageError, parse_cli, verb, *args)
self.assertIn("option %s not recognized" % (option,), str(e))
def test_create_client_config(self):
d = self.mktemp()
os.mkdir(d)
fname = os.path.join(d, 'tahoe.cfg')
with open(fname, 'w') as f:
opts = {"nickname": "nick",
"webport": "tcp:3456",
"hide-ip": False,
"listen": "none",
"shares-needed": "1",
"shares-happy": "1",
"shares-total": "1",
}
create_node.write_node_config(f, opts)
create_node.write_client_config(f, opts)
config = configutil.get_config(fname)
# should succeed, no exceptions
configutil.validate_config(
fname,
config,
client._valid_config(),
)
@defer.inlineCallbacks
def test_client(self):
basedir = self.mktemp()

105
src/allmydata/test/test_configutil.py
View File

@ -1,14 +1,26 @@
"""
Tests for allmydata.util.configutil.
Ported to Python 3.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
from future.utils import PY2
if PY2:
# Omitted dict, cause worried about interactions.
from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, list, object, range, str, max, min # noqa: F401
import os.path
from twisted.trial import unittest
from allmydata.util import configutil
from allmydata.test.no_network import GridTestMixin
from ..scripts import create_node
from .. import client
class ConfigUtilTests(GridTestMixin, unittest.TestCase):
class ConfigUtilTests(unittest.TestCase):
def setUp(self):
super(ConfigUtilTests, self).setUp()
self.static_valid_config = configutil.ValidConfiguration(
@ -20,10 +32,22 @@ class ConfigUtilTests(GridTestMixin, unittest.TestCase):
lambda section_name, item_name: (section_name, item_name) == ("node", "valid"),
)
def create_tahoe_cfg(self, cfg):
d = self.mktemp()
os.mkdir(d)
fname = os.path.join(d, 'tahoe.cfg')
with open(fname, "w") as f:
f.write(cfg)
return fname
def test_config_utils(self):
self.basedir = "cli/ConfigUtilTests/test-config-utils"
self.set_up_grid(oneshare=True)
tahoe_cfg = os.path.join(self.get_clientdir(i=0), "tahoe.cfg")
tahoe_cfg = self.create_tahoe_cfg("""\
[node]
nickname = client-0
web.port = adopt-socket:fd=5
[storage]
enabled = false
""")
# test that at least one option was read correctly
config = configutil.get_config(tahoe_cfg)
@ -45,12 +69,7 @@ class ConfigUtilTests(GridTestMixin, unittest.TestCase):
self.failUnlessEqual(config.get("node", "descriptor"), descriptor)
def test_config_validation_success(self):
d = self.mktemp()
os.mkdir(d)
fname = os.path.join(d, 'tahoe.cfg')
with open(fname, 'w') as f:
f.write('[node]\nvalid = foo\n')
fname = self.create_tahoe_cfg('[node]\nvalid = foo\n')
config = configutil.get_config(fname)
# should succeed, no exceptions
@ -66,12 +85,7 @@ class ConfigUtilTests(GridTestMixin, unittest.TestCase):
validation but are matched by the dynamic validation is considered
valid.
"""
d = self.mktemp()
os.mkdir(d)
fname = os.path.join(d, 'tahoe.cfg')
with open(fname, 'w') as f:
f.write('[node]\nvalid = foo\n')
fname = self.create_tahoe_cfg('[node]\nvalid = foo\n')
config = configutil.get_config(fname)
# should succeed, no exceptions
@ -82,12 +96,7 @@ class ConfigUtilTests(GridTestMixin, unittest.TestCase):
)
def test_config_validation_invalid_item(self):
d = self.mktemp()
os.mkdir(d)
fname = os.path.join(d, 'tahoe.cfg')
with open(fname, 'w') as f:
f.write('[node]\nvalid = foo\ninvalid = foo\n')
fname = self.create_tahoe_cfg('[node]\nvalid = foo\ninvalid = foo\n')
config = configutil.get_config(fname)
e = self.assertRaises(
@ -103,12 +112,7 @@ class ConfigUtilTests(GridTestMixin, unittest.TestCase):
A configuration with a section that is matched by neither the static nor
dynamic validators is rejected.
"""
d = self.mktemp()
os.mkdir(d)
fname = os.path.join(d, 'tahoe.cfg')
with open(fname, 'w') as f:
f.write('[node]\nvalid = foo\n[invalid]\n')
fname = self.create_tahoe_cfg('[node]\nvalid = foo\n[invalid]\n')
config = configutil.get_config(fname)
e = self.assertRaises(
@ -124,12 +128,7 @@ class ConfigUtilTests(GridTestMixin, unittest.TestCase):
A configuration with a section that is matched by neither the static nor
dynamic validators is rejected.
"""
d = self.mktemp()
os.mkdir(d)
fname = os.path.join(d, 'tahoe.cfg')
with open(fname, 'w') as f:
f.write('[node]\nvalid = foo\n[invalid]\n')
fname = self.create_tahoe_cfg('[node]\nvalid = foo\n[invalid]\n')
config = configutil.get_config(fname)
e = self.assertRaises(
@ -145,12 +144,7 @@ class ConfigUtilTests(GridTestMixin, unittest.TestCase):
A configuration with a section, item pair that is matched by neither the
static nor dynamic validators is rejected.
"""
d = self.mktemp()
os.mkdir(d)
fname = os.path.join(d, 'tahoe.cfg')
with open(fname, 'w') as f:
f.write('[node]\nvalid = foo\ninvalid = foo\n')
fname = self.create_tahoe_cfg('[node]\nvalid = foo\ninvalid = foo\n')
config = configutil.get_config(fname)
e = self.assertRaises(
@ -160,28 +154,3 @@ class ConfigUtilTests(GridTestMixin, unittest.TestCase):
self.dynamic_valid_config,
)
self.assertIn("section [node] contains unknown option 'invalid'", str(e))
def test_create_client_config(self):
d = self.mktemp()
os.mkdir(d)
fname = os.path.join(d, 'tahoe.cfg')
with open(fname, 'w') as f:
opts = {"nickname": "nick",
"webport": "tcp:3456",
"hide-ip": False,
"listen": "none",
"shares-needed": "1",
"shares-happy": "1",
"shares-total": "1",
}
create_node.write_node_config(f, opts)
create_node.write_client_config(f, opts)
config = configutil.get_config(fname)
# should succeed, no exceptions
configutil.validate_config(
fname,
config,
client._valid_config(),
)

122
src/allmydata/test/test_connection_status.py Normal file
View File

@ -0,0 +1,122 @@
"""
Tests for allmydata.util.connection_status.
Port to Python 3.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
from future.utils import PY2
if PY2:
from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
import mock
from twisted.trial import unittest
from ..util import connection_status
class Status(unittest.TestCase):
def test_hint_statuses(self):
ncs = connection_status._hint_statuses(["h2","h1"],
{"h1": "hand1", "h4": "hand4"},
{"h1": "st1", "h2": "st2",
"h3": "st3"})
self.assertEqual(ncs, {"h1 via hand1": "st1",
"h2": "st2"})
def test_reconnector_connected(self):
ci = mock.Mock()
ci.connectorStatuses = {"h1": "st1"}
ci.connectionHandlers = {"h1": "hand1"}
ci.winningHint = "h1"
ci.establishedAt = 120
ri = mock.Mock()
ri.state = "connected"
ri.connectionInfo = ci
rc = mock.Mock
rc.getReconnectionInfo = mock.Mock(return_value=ri)
cs = connection_status.from_foolscap_reconnector(rc, 123)
self.assertEqual(cs.connected, True)
self.assertEqual(cs.summary, "Connected to h1 via hand1")
self.assertEqual(cs.non_connected_statuses, {})
self.assertEqual(cs.last_connection_time, 120)
self.assertEqual(cs.last_received_time, 123)
def test_reconnector_connected_others(self):
ci = mock.Mock()
ci.connectorStatuses = {"h1": "st1", "h2": "st2"}
ci.connectionHandlers = {"h1": "hand1"}
ci.winningHint = "h1"
ci.establishedAt = 120
ri = mock.Mock()
ri.state = "connected"
ri.connectionInfo = ci
rc = mock.Mock
rc.getReconnectionInfo = mock.Mock(return_value=ri)
cs = connection_status.from_foolscap_reconnector(rc, 123)
self.assertEqual(cs.connected, True)
self.assertEqual(cs.summary, "Connected to h1 via hand1")
self.assertEqual(cs.non_connected_statuses, {"h2": "st2"})
self.assertEqual(cs.last_connection_time, 120)
self.assertEqual(cs.last_received_time, 123)
def test_reconnector_connected_listener(self):
ci = mock.Mock()
ci.connectorStatuses = {"h1": "st1", "h2": "st2"}
ci.connectionHandlers = {"h1": "hand1"}
ci.listenerStatus = ("listener1", "successful")
ci.winningHint = None
ci.establishedAt = 120
ri = mock.Mock()
ri.state = "connected"
ri.connectionInfo = ci
rc = mock.Mock
rc.getReconnectionInfo = mock.Mock(return_value=ri)
cs = connection_status.from_foolscap_reconnector(rc, 123)
self.assertEqual(cs.connected, True)
self.assertEqual(cs.summary, "Connected via listener (listener1)")
self.assertEqual(cs.non_connected_statuses,
{"h1 via hand1": "st1", "h2": "st2"})
self.assertEqual(cs.last_connection_time, 120)
self.assertEqual(cs.last_received_time, 123)
def test_reconnector_connecting(self):
ci = mock.Mock()
ci.connectorStatuses = {"h1": "st1", "h2": "st2"}
ci.connectionHandlers = {"h1": "hand1"}
ri = mock.Mock()
ri.state = "connecting"
ri.connectionInfo = ci
rc = mock.Mock
rc.getReconnectionInfo = mock.Mock(return_value=ri)
cs = connection_status.from_foolscap_reconnector(rc, 123)
self.assertEqual(cs.connected, False)
self.assertEqual(cs.summary, "Trying to connect")
self.assertEqual(cs.non_connected_statuses,
{"h1 via hand1": "st1", "h2": "st2"})
self.assertEqual(cs.last_connection_time, None)
self.assertEqual(cs.last_received_time, 123)
def test_reconnector_waiting(self):
ci = mock.Mock()
ci.connectorStatuses = {"h1": "st1", "h2": "st2"}
ci.connectionHandlers = {"h1": "hand1"}
ri = mock.Mock()
ri.state = "waiting"
ri.lastAttempt = 10
ri.nextAttempt = 20
ri.connectionInfo = ci
rc = mock.Mock
rc.getReconnectionInfo = mock.Mock(return_value=ri)
with mock.patch("time.time", return_value=12):
cs = connection_status.from_foolscap_reconnector(rc, 5)
self.assertEqual(cs.connected, False)
self.assertEqual(cs.summary,
"Reconnecting in 8 seconds (last attempt 2s ago)")
self.assertEqual(cs.non_connected_statuses,
{"h1 via hand1": "st1", "h2": "st2"})
self.assertEqual(cs.last_connection_time, None)
self.assertEqual(cs.last_received_time, 5)

104
src/allmydata/test/test_connections.py
View File

@ -7,7 +7,6 @@ from foolscap.connections import tcp
from ..node import PrivacyError, config_from_string
from ..node import create_connection_handlers
from ..node import create_main_tub, _tub_portlocation
from ..util import connection_status
from ..util.i2p_provider import create as create_i2p_provider
from ..util.tor_provider import create as create_tor_provider
@ -463,106 +462,3 @@ class Privacy(unittest.TestCase):
str(ctx.exception),
"tub.location includes tcp: hint",
)
class Status(unittest.TestCase):
def test_hint_statuses(self):
ncs = connection_status._hint_statuses(["h2","h1"],
{"h1": "hand1", "h4": "hand4"},
{"h1": "st1", "h2": "st2",
"h3": "st3"})
self.assertEqual(ncs, {"h1 via hand1": "st1",
"h2": "st2"})
def test_reconnector_connected(self):
ci = mock.Mock()
ci.connectorStatuses = {"h1": "st1"}
ci.connectionHandlers = {"h1": "hand1"}
ci.winningHint = "h1"
ci.establishedAt = 120
ri = mock.Mock()
ri.state = "connected"
ri.connectionInfo = ci
rc = mock.Mock
rc.getReconnectionInfo = mock.Mock(return_value=ri)
cs = connection_status.from_foolscap_reconnector(rc, 123)
self.assertEqual(cs.connected, True)
self.assertEqual(cs.summary, "Connected to h1 via hand1")
self.assertEqual(cs.non_connected_statuses, {})
self.assertEqual(cs.last_connection_time, 120)
self.assertEqual(cs.last_received_time, 123)
def test_reconnector_connected_others(self):
ci = mock.Mock()
ci.connectorStatuses = {"h1": "st1", "h2": "st2"}
ci.connectionHandlers = {"h1": "hand1"}
ci.winningHint = "h1"
ci.establishedAt = 120
ri = mock.Mock()
ri.state = "connected"
ri.connectionInfo = ci
rc = mock.Mock
rc.getReconnectionInfo = mock.Mock(return_value=ri)
cs = connection_status.from_foolscap_reconnector(rc, 123)
self.assertEqual(cs.connected, True)
self.assertEqual(cs.summary, "Connected to h1 via hand1")
self.assertEqual(cs.non_connected_statuses, {"h2": "st2"})
self.assertEqual(cs.last_connection_time, 120)
self.assertEqual(cs.last_received_time, 123)
def test_reconnector_connected_listener(self):
ci = mock.Mock()
ci.connectorStatuses = {"h1": "st1", "h2": "st2"}
ci.connectionHandlers = {"h1": "hand1"}
ci.listenerStatus = ("listener1", "successful")
ci.winningHint = None
ci.establishedAt = 120
ri = mock.Mock()
ri.state = "connected"
ri.connectionInfo = ci
rc = mock.Mock
rc.getReconnectionInfo = mock.Mock(return_value=ri)
cs = connection_status.from_foolscap_reconnector(rc, 123)
self.assertEqual(cs.connected, True)
self.assertEqual(cs.summary, "Connected via listener (listener1)")
self.assertEqual(cs.non_connected_statuses,
{"h1 via hand1": "st1", "h2": "st2"})
self.assertEqual(cs.last_connection_time, 120)
self.assertEqual(cs.last_received_time, 123)
def test_reconnector_connecting(self):
ci = mock.Mock()
ci.connectorStatuses = {"h1": "st1", "h2": "st2"}
ci.connectionHandlers = {"h1": "hand1"}
ri = mock.Mock()
ri.state = "connecting"
ri.connectionInfo = ci
rc = mock.Mock
rc.getReconnectionInfo = mock.Mock(return_value=ri)
cs = connection_status.from_foolscap_reconnector(rc, 123)
self.assertEqual(cs.connected, False)
self.assertEqual(cs.summary, "Trying to connect")
self.assertEqual(cs.non_connected_statuses,
{"h1 via hand1": "st1", "h2": "st2"})
self.assertEqual(cs.last_connection_time, None)
self.assertEqual(cs.last_received_time, 123)
def test_reconnector_waiting(self):
ci = mock.Mock()
ci.connectorStatuses = {"h1": "st1", "h2": "st2"}
ci.connectionHandlers = {"h1": "hand1"}
ri = mock.Mock()
ri.state = "waiting"
ri.lastAttempt = 10
ri.nextAttempt = 20
ri.connectionInfo = ci
rc = mock.Mock
rc.getReconnectionInfo = mock.Mock(return_value=ri)
with mock.patch("time.time", return_value=12):
cs = connection_status.from_foolscap_reconnector(rc, 5)
self.assertEqual(cs.connected, False)
self.assertEqual(cs.summary,
"Reconnecting in 8 seconds (last attempt 2s ago)")
self.assertEqual(cs.non_connected_statuses,
{"h1 via hand1": "st1", "h2": "st2"})
self.assertEqual(cs.last_connection_time, None)
self.assertEqual(cs.last_received_time, 5)

4
src/allmydata/util/_python3.py
View File

@ -41,6 +41,8 @@ PORTED_MODULES = [
"allmydata.util.assertutil",
"allmydata.util.base32",
"allmydata.util.base62",
"allmydata.util.configutil",
"allmydata.util.connection_status",
"allmydata.util.deferredutil",
"allmydata.util.fileutil",
"allmydata.util.dictutil",
@ -66,6 +68,8 @@ PORTED_TEST_MODULES = [
"allmydata.test.test_abbreviate",
"allmydata.test.test_base32",
"allmydata.test.test_base62",
"allmydata.test.test_configutil",
"allmydata.test.test_connection_status",
"allmydata.test.test_crawler",
"allmydata.test.test_crypto",
"allmydata.test.test_deferredutil",

39
src/allmydata/util/configutil.py
View File

@ -1,8 +1,32 @@
"""
Read/write config files.
Configuration is returned as native strings.
Ported to Python 3.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
from future.utils import PY2
if PY2:
# We don't do open(), because we want files to read/write native strs when
# we do "r" or "w".
from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
if PY2:
# In theory on Python 2 configparser also works, but then code gets the
# wrong exceptions and they don't get handled. So just use native parser
# for now.
from ConfigParser import SafeConfigParser
else:
from configparser import SafeConfigParser
import attr
class UnknownConfigError(Exception):
"""
An unknown config item was found.
@ -12,11 +36,16 @@ class UnknownConfigError(Exception):
def get_config(tahoe_cfg):
"""Load the config, returning a SafeConfigParser.
Configuration is returned as native strings.
"""
config = SafeConfigParser()
with open(tahoe_cfg, "rb") as f:
# Skip any initial Byte Order Mark. Since this is an ordinary file, we
# don't need to handle incomplete reads, and can assume seekability.
if f.read(3) != '\xEF\xBB\xBF':
with open(tahoe_cfg, "r") as f:
# On Python 2, where we read in bytes, skip any initial Byte Order
# Mark. Since this is an ordinary file, we don't need to handle
# incomplete reads, and can assume seekability.
if PY2 and f.read(3) != b'\xEF\xBB\xBF':
f.seek(0)
config.readfp(f)
return config
@ -28,7 +57,7 @@ def set_config(config, section, option, value):
assert config.get(section, option) == value
def write_config(tahoe_cfg, config):
with open(tahoe_cfg, "wb") as f:
with open(tahoe_cfg, "w") as f:
config.write(f)
def validate_config(fname, cfg, valid_config):

22
src/allmydata/util/connection_status.py
View File

@ -1,3 +1,18 @@
"""
Parse connection status from Foolscap.
Ported to Python 3.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
from future.utils import PY2
if PY2:
from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
import time
from zope.interface import implementer
from ..interfaces import IConnectionStatus
@ -37,9 +52,12 @@ def _hint_statuses(which, handlers, statuses):
def from_foolscap_reconnector(rc, last_received):
ri = rc.getReconnectionInfo()
# See foolscap/reconnector.py, ReconnectionInfo, for details about
# possible states.
# See foolscap/reconnector.py, ReconnectionInfo, for details about possible
# states. The returned result is a native string, it seems, so convert to
# unicode.
state = ri.state
if isinstance(state, bytes): # Python 2
state = str(state, "ascii")
if state == "unstarted":
return ConnectionStatus.unstarted()