diff --git a/newsfragments/3377.minor b/newsfragments/3377.minor new file mode 100644 index 000000000..e69de29bb diff --git a/src/allmydata/interfaces.py b/src/allmydata/interfaces.py index 081d9a33d..c93c1d81d 100644 --- a/src/allmydata/interfaces.py +++ b/src/allmydata/interfaces.py @@ -1,4 +1,3 @@ - from past.builtins import long from zope.interface import Interface, Attribute diff --git a/src/allmydata/test/cli/test_create.py b/src/allmydata/test/cli/test_create.py index 75162c39e..f013c0205 100644 --- a/src/allmydata/test/cli/test_create.py +++ b/src/allmydata/test/cli/test_create.py @@ -6,6 +6,8 @@ from twisted.python import usage from allmydata.util import configutil from ..common_util import run_cli, parse_cli from ...scripts import create_node +from ... import client + def read_config(basedir): tahoe_cfg = os.path.join(basedir, "tahoe.cfg") @@ -33,6 +35,31 @@ class Config(unittest.TestCase): e = self.assertRaises(usage.UsageError, parse_cli, verb, *args) self.assertIn("option %s not recognized" % (option,), str(e)) + def test_create_client_config(self): + d = self.mktemp() + os.mkdir(d) + fname = os.path.join(d, 'tahoe.cfg') + + with open(fname, 'w') as f: + opts = {"nickname": "nick", + "webport": "tcp:3456", + "hide-ip": False, + "listen": "none", + "shares-needed": "1", + "shares-happy": "1", + "shares-total": "1", + } + create_node.write_node_config(f, opts) + create_node.write_client_config(f, opts) + + config = configutil.get_config(fname) + # should succeed, no exceptions + configutil.validate_config( + fname, + config, + client._valid_config(), + ) + @defer.inlineCallbacks def test_client(self): basedir = self.mktemp() diff --git a/src/allmydata/test/test_configutil.py b/src/allmydata/test/test_configutil.py index 45eb6ac25..c57381289 100644 --- a/src/allmydata/test/test_configutil.py +++ b/src/allmydata/test/test_configutil.py @@ -1,14 +1,26 @@ +""" +Tests for allmydata.util.configutil. + +Ported to Python 3. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + # Omitted dict, cause worried about interactions. + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, list, object, range, str, max, min # noqa: F401 + import os.path from twisted.trial import unittest from allmydata.util import configutil -from allmydata.test.no_network import GridTestMixin -from ..scripts import create_node -from .. import client -class ConfigUtilTests(GridTestMixin, unittest.TestCase): +class ConfigUtilTests(unittest.TestCase): def setUp(self): super(ConfigUtilTests, self).setUp() self.static_valid_config = configutil.ValidConfiguration( @@ -20,10 +32,22 @@ class ConfigUtilTests(GridTestMixin, unittest.TestCase): lambda section_name, item_name: (section_name, item_name) == ("node", "valid"), ) + def create_tahoe_cfg(self, cfg): + d = self.mktemp() + os.mkdir(d) + fname = os.path.join(d, 'tahoe.cfg') + with open(fname, "w") as f: + f.write(cfg) + return fname + def test_config_utils(self): - self.basedir = "cli/ConfigUtilTests/test-config-utils" - self.set_up_grid(oneshare=True) - tahoe_cfg = os.path.join(self.get_clientdir(i=0), "tahoe.cfg") + tahoe_cfg = self.create_tahoe_cfg("""\ +[node] +nickname = client-0 +web.port = adopt-socket:fd=5 +[storage] +enabled = false +""") # test that at least one option was read correctly config = configutil.get_config(tahoe_cfg) @@ -45,12 +69,7 @@ class ConfigUtilTests(GridTestMixin, unittest.TestCase): self.failUnlessEqual(config.get("node", "descriptor"), descriptor) def test_config_validation_success(self): - d = self.mktemp() - os.mkdir(d) - fname = os.path.join(d, 'tahoe.cfg') - - with open(fname, 'w') as f: - f.write('[node]\nvalid = foo\n') + fname = self.create_tahoe_cfg('[node]\nvalid = foo\n') config = configutil.get_config(fname) # should succeed, no exceptions @@ -66,12 +85,7 @@ class ConfigUtilTests(GridTestMixin, unittest.TestCase): validation but are matched by the dynamic validation is considered valid. """ - d = self.mktemp() - os.mkdir(d) - fname = os.path.join(d, 'tahoe.cfg') - - with open(fname, 'w') as f: - f.write('[node]\nvalid = foo\n') + fname = self.create_tahoe_cfg('[node]\nvalid = foo\n') config = configutil.get_config(fname) # should succeed, no exceptions @@ -82,12 +96,7 @@ class ConfigUtilTests(GridTestMixin, unittest.TestCase): ) def test_config_validation_invalid_item(self): - d = self.mktemp() - os.mkdir(d) - fname = os.path.join(d, 'tahoe.cfg') - - with open(fname, 'w') as f: - f.write('[node]\nvalid = foo\ninvalid = foo\n') + fname = self.create_tahoe_cfg('[node]\nvalid = foo\ninvalid = foo\n') config = configutil.get_config(fname) e = self.assertRaises( @@ -103,12 +112,7 @@ class ConfigUtilTests(GridTestMixin, unittest.TestCase): A configuration with a section that is matched by neither the static nor dynamic validators is rejected. """ - d = self.mktemp() - os.mkdir(d) - fname = os.path.join(d, 'tahoe.cfg') - - with open(fname, 'w') as f: - f.write('[node]\nvalid = foo\n[invalid]\n') + fname = self.create_tahoe_cfg('[node]\nvalid = foo\n[invalid]\n') config = configutil.get_config(fname) e = self.assertRaises( @@ -124,12 +128,7 @@ class ConfigUtilTests(GridTestMixin, unittest.TestCase): A configuration with a section that is matched by neither the static nor dynamic validators is rejected. """ - d = self.mktemp() - os.mkdir(d) - fname = os.path.join(d, 'tahoe.cfg') - - with open(fname, 'w') as f: - f.write('[node]\nvalid = foo\n[invalid]\n') + fname = self.create_tahoe_cfg('[node]\nvalid = foo\n[invalid]\n') config = configutil.get_config(fname) e = self.assertRaises( @@ -145,12 +144,7 @@ class ConfigUtilTests(GridTestMixin, unittest.TestCase): A configuration with a section, item pair that is matched by neither the static nor dynamic validators is rejected. """ - d = self.mktemp() - os.mkdir(d) - fname = os.path.join(d, 'tahoe.cfg') - - with open(fname, 'w') as f: - f.write('[node]\nvalid = foo\ninvalid = foo\n') + fname = self.create_tahoe_cfg('[node]\nvalid = foo\ninvalid = foo\n') config = configutil.get_config(fname) e = self.assertRaises( @@ -160,28 +154,3 @@ class ConfigUtilTests(GridTestMixin, unittest.TestCase): self.dynamic_valid_config, ) self.assertIn("section [node] contains unknown option 'invalid'", str(e)) - - def test_create_client_config(self): - d = self.mktemp() - os.mkdir(d) - fname = os.path.join(d, 'tahoe.cfg') - - with open(fname, 'w') as f: - opts = {"nickname": "nick", - "webport": "tcp:3456", - "hide-ip": False, - "listen": "none", - "shares-needed": "1", - "shares-happy": "1", - "shares-total": "1", - } - create_node.write_node_config(f, opts) - create_node.write_client_config(f, opts) - - config = configutil.get_config(fname) - # should succeed, no exceptions - configutil.validate_config( - fname, - config, - client._valid_config(), - ) diff --git a/src/allmydata/test/test_connection_status.py b/src/allmydata/test/test_connection_status.py new file mode 100644 index 000000000..2bd8bf6ab --- /dev/null +++ b/src/allmydata/test/test_connection_status.py @@ -0,0 +1,122 @@ +""" +Tests for allmydata.util.connection_status. + +Port to Python 3. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + +import mock + +from twisted.trial import unittest + +from ..util import connection_status + +class Status(unittest.TestCase): + def test_hint_statuses(self): + ncs = connection_status._hint_statuses(["h2","h1"], + {"h1": "hand1", "h4": "hand4"}, + {"h1": "st1", "h2": "st2", + "h3": "st3"}) + self.assertEqual(ncs, {"h1 via hand1": "st1", + "h2": "st2"}) + + def test_reconnector_connected(self): + ci = mock.Mock() + ci.connectorStatuses = {"h1": "st1"} + ci.connectionHandlers = {"h1": "hand1"} + ci.winningHint = "h1" + ci.establishedAt = 120 + ri = mock.Mock() + ri.state = "connected" + ri.connectionInfo = ci + rc = mock.Mock + rc.getReconnectionInfo = mock.Mock(return_value=ri) + cs = connection_status.from_foolscap_reconnector(rc, 123) + self.assertEqual(cs.connected, True) + self.assertEqual(cs.summary, "Connected to h1 via hand1") + self.assertEqual(cs.non_connected_statuses, {}) + self.assertEqual(cs.last_connection_time, 120) + self.assertEqual(cs.last_received_time, 123) + + def test_reconnector_connected_others(self): + ci = mock.Mock() + ci.connectorStatuses = {"h1": "st1", "h2": "st2"} + ci.connectionHandlers = {"h1": "hand1"} + ci.winningHint = "h1" + ci.establishedAt = 120 + ri = mock.Mock() + ri.state = "connected" + ri.connectionInfo = ci + rc = mock.Mock + rc.getReconnectionInfo = mock.Mock(return_value=ri) + cs = connection_status.from_foolscap_reconnector(rc, 123) + self.assertEqual(cs.connected, True) + self.assertEqual(cs.summary, "Connected to h1 via hand1") + self.assertEqual(cs.non_connected_statuses, {"h2": "st2"}) + self.assertEqual(cs.last_connection_time, 120) + self.assertEqual(cs.last_received_time, 123) + + def test_reconnector_connected_listener(self): + ci = mock.Mock() + ci.connectorStatuses = {"h1": "st1", "h2": "st2"} + ci.connectionHandlers = {"h1": "hand1"} + ci.listenerStatus = ("listener1", "successful") + ci.winningHint = None + ci.establishedAt = 120 + ri = mock.Mock() + ri.state = "connected" + ri.connectionInfo = ci + rc = mock.Mock + rc.getReconnectionInfo = mock.Mock(return_value=ri) + cs = connection_status.from_foolscap_reconnector(rc, 123) + self.assertEqual(cs.connected, True) + self.assertEqual(cs.summary, "Connected via listener (listener1)") + self.assertEqual(cs.non_connected_statuses, + {"h1 via hand1": "st1", "h2": "st2"}) + self.assertEqual(cs.last_connection_time, 120) + self.assertEqual(cs.last_received_time, 123) + + def test_reconnector_connecting(self): + ci = mock.Mock() + ci.connectorStatuses = {"h1": "st1", "h2": "st2"} + ci.connectionHandlers = {"h1": "hand1"} + ri = mock.Mock() + ri.state = "connecting" + ri.connectionInfo = ci + rc = mock.Mock + rc.getReconnectionInfo = mock.Mock(return_value=ri) + cs = connection_status.from_foolscap_reconnector(rc, 123) + self.assertEqual(cs.connected, False) + self.assertEqual(cs.summary, "Trying to connect") + self.assertEqual(cs.non_connected_statuses, + {"h1 via hand1": "st1", "h2": "st2"}) + self.assertEqual(cs.last_connection_time, None) + self.assertEqual(cs.last_received_time, 123) + + def test_reconnector_waiting(self): + ci = mock.Mock() + ci.connectorStatuses = {"h1": "st1", "h2": "st2"} + ci.connectionHandlers = {"h1": "hand1"} + ri = mock.Mock() + ri.state = "waiting" + ri.lastAttempt = 10 + ri.nextAttempt = 20 + ri.connectionInfo = ci + rc = mock.Mock + rc.getReconnectionInfo = mock.Mock(return_value=ri) + with mock.patch("time.time", return_value=12): + cs = connection_status.from_foolscap_reconnector(rc, 5) + self.assertEqual(cs.connected, False) + self.assertEqual(cs.summary, + "Reconnecting in 8 seconds (last attempt 2s ago)") + self.assertEqual(cs.non_connected_statuses, + {"h1 via hand1": "st1", "h2": "st2"}) + self.assertEqual(cs.last_connection_time, None) + self.assertEqual(cs.last_received_time, 5) diff --git a/src/allmydata/test/test_connections.py b/src/allmydata/test/test_connections.py index 3e2806dd0..9b5bd7f30 100644 --- a/src/allmydata/test/test_connections.py +++ b/src/allmydata/test/test_connections.py @@ -7,7 +7,6 @@ from foolscap.connections import tcp from ..node import PrivacyError, config_from_string from ..node import create_connection_handlers from ..node import create_main_tub, _tub_portlocation -from ..util import connection_status from ..util.i2p_provider import create as create_i2p_provider from ..util.tor_provider import create as create_tor_provider @@ -463,106 +462,3 @@ class Privacy(unittest.TestCase): str(ctx.exception), "tub.location includes tcp: hint", ) - -class Status(unittest.TestCase): - def test_hint_statuses(self): - ncs = connection_status._hint_statuses(["h2","h1"], - {"h1": "hand1", "h4": "hand4"}, - {"h1": "st1", "h2": "st2", - "h3": "st3"}) - self.assertEqual(ncs, {"h1 via hand1": "st1", - "h2": "st2"}) - - def test_reconnector_connected(self): - ci = mock.Mock() - ci.connectorStatuses = {"h1": "st1"} - ci.connectionHandlers = {"h1": "hand1"} - ci.winningHint = "h1" - ci.establishedAt = 120 - ri = mock.Mock() - ri.state = "connected" - ri.connectionInfo = ci - rc = mock.Mock - rc.getReconnectionInfo = mock.Mock(return_value=ri) - cs = connection_status.from_foolscap_reconnector(rc, 123) - self.assertEqual(cs.connected, True) - self.assertEqual(cs.summary, "Connected to h1 via hand1") - self.assertEqual(cs.non_connected_statuses, {}) - self.assertEqual(cs.last_connection_time, 120) - self.assertEqual(cs.last_received_time, 123) - - def test_reconnector_connected_others(self): - ci = mock.Mock() - ci.connectorStatuses = {"h1": "st1", "h2": "st2"} - ci.connectionHandlers = {"h1": "hand1"} - ci.winningHint = "h1" - ci.establishedAt = 120 - ri = mock.Mock() - ri.state = "connected" - ri.connectionInfo = ci - rc = mock.Mock - rc.getReconnectionInfo = mock.Mock(return_value=ri) - cs = connection_status.from_foolscap_reconnector(rc, 123) - self.assertEqual(cs.connected, True) - self.assertEqual(cs.summary, "Connected to h1 via hand1") - self.assertEqual(cs.non_connected_statuses, {"h2": "st2"}) - self.assertEqual(cs.last_connection_time, 120) - self.assertEqual(cs.last_received_time, 123) - - def test_reconnector_connected_listener(self): - ci = mock.Mock() - ci.connectorStatuses = {"h1": "st1", "h2": "st2"} - ci.connectionHandlers = {"h1": "hand1"} - ci.listenerStatus = ("listener1", "successful") - ci.winningHint = None - ci.establishedAt = 120 - ri = mock.Mock() - ri.state = "connected" - ri.connectionInfo = ci - rc = mock.Mock - rc.getReconnectionInfo = mock.Mock(return_value=ri) - cs = connection_status.from_foolscap_reconnector(rc, 123) - self.assertEqual(cs.connected, True) - self.assertEqual(cs.summary, "Connected via listener (listener1)") - self.assertEqual(cs.non_connected_statuses, - {"h1 via hand1": "st1", "h2": "st2"}) - self.assertEqual(cs.last_connection_time, 120) - self.assertEqual(cs.last_received_time, 123) - - def test_reconnector_connecting(self): - ci = mock.Mock() - ci.connectorStatuses = {"h1": "st1", "h2": "st2"} - ci.connectionHandlers = {"h1": "hand1"} - ri = mock.Mock() - ri.state = "connecting" - ri.connectionInfo = ci - rc = mock.Mock - rc.getReconnectionInfo = mock.Mock(return_value=ri) - cs = connection_status.from_foolscap_reconnector(rc, 123) - self.assertEqual(cs.connected, False) - self.assertEqual(cs.summary, "Trying to connect") - self.assertEqual(cs.non_connected_statuses, - {"h1 via hand1": "st1", "h2": "st2"}) - self.assertEqual(cs.last_connection_time, None) - self.assertEqual(cs.last_received_time, 123) - - def test_reconnector_waiting(self): - ci = mock.Mock() - ci.connectorStatuses = {"h1": "st1", "h2": "st2"} - ci.connectionHandlers = {"h1": "hand1"} - ri = mock.Mock() - ri.state = "waiting" - ri.lastAttempt = 10 - ri.nextAttempt = 20 - ri.connectionInfo = ci - rc = mock.Mock - rc.getReconnectionInfo = mock.Mock(return_value=ri) - with mock.patch("time.time", return_value=12): - cs = connection_status.from_foolscap_reconnector(rc, 5) - self.assertEqual(cs.connected, False) - self.assertEqual(cs.summary, - "Reconnecting in 8 seconds (last attempt 2s ago)") - self.assertEqual(cs.non_connected_statuses, - {"h1 via hand1": "st1", "h2": "st2"}) - self.assertEqual(cs.last_connection_time, None) - self.assertEqual(cs.last_received_time, 5) diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 82ddebc72..2ec91ed96 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -41,6 +41,8 @@ PORTED_MODULES = [ "allmydata.util.assertutil", "allmydata.util.base32", "allmydata.util.base62", + "allmydata.util.configutil", + "allmydata.util.connection_status", "allmydata.util.deferredutil", "allmydata.util.fileutil", "allmydata.util.dictutil", @@ -66,6 +68,8 @@ PORTED_TEST_MODULES = [ "allmydata.test.test_abbreviate", "allmydata.test.test_base32", "allmydata.test.test_base62", + "allmydata.test.test_configutil", + "allmydata.test.test_connection_status", "allmydata.test.test_crawler", "allmydata.test.test_crypto", "allmydata.test.test_deferredutil", diff --git a/src/allmydata/util/configutil.py b/src/allmydata/util/configutil.py index 3699db35d..1a1a93f18 100644 --- a/src/allmydata/util/configutil.py +++ b/src/allmydata/util/configutil.py @@ -1,8 +1,32 @@ +""" +Read/write config files. -from ConfigParser import SafeConfigParser +Configuration is returned as native strings. + +Ported to Python 3. +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + # We don't do open(), because we want files to read/write native strs when + # we do "r" or "w". + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + +if PY2: + # In theory on Python 2 configparser also works, but then code gets the + # wrong exceptions and they don't get handled. So just use native parser + # for now. + from ConfigParser import SafeConfigParser +else: + from configparser import SafeConfigParser import attr + class UnknownConfigError(Exception): """ An unknown config item was found. @@ -12,11 +36,16 @@ class UnknownConfigError(Exception): def get_config(tahoe_cfg): + """Load the config, returning a SafeConfigParser. + + Configuration is returned as native strings. + """ config = SafeConfigParser() - with open(tahoe_cfg, "rb") as f: - # Skip any initial Byte Order Mark. Since this is an ordinary file, we - # don't need to handle incomplete reads, and can assume seekability. - if f.read(3) != '\xEF\xBB\xBF': + with open(tahoe_cfg, "r") as f: + # On Python 2, where we read in bytes, skip any initial Byte Order + # Mark. Since this is an ordinary file, we don't need to handle + # incomplete reads, and can assume seekability. + if PY2 and f.read(3) != b'\xEF\xBB\xBF': f.seek(0) config.readfp(f) return config @@ -28,7 +57,7 @@ def set_config(config, section, option, value): assert config.get(section, option) == value def write_config(tahoe_cfg, config): - with open(tahoe_cfg, "wb") as f: + with open(tahoe_cfg, "w") as f: config.write(f) def validate_config(fname, cfg, valid_config): diff --git a/src/allmydata/util/connection_status.py b/src/allmydata/util/connection_status.py index 44c12f220..0e8595e81 100644 --- a/src/allmydata/util/connection_status.py +++ b/src/allmydata/util/connection_status.py @@ -1,3 +1,18 @@ +""" +Parse connection status from Foolscap. + +Ported to Python 3. +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + import time from zope.interface import implementer from ..interfaces import IConnectionStatus @@ -37,9 +52,12 @@ def _hint_statuses(which, handlers, statuses): def from_foolscap_reconnector(rc, last_received): ri = rc.getReconnectionInfo() - # See foolscap/reconnector.py, ReconnectionInfo, for details about - # possible states. + # See foolscap/reconnector.py, ReconnectionInfo, for details about possible + # states. The returned result is a native string, it seems, so convert to + # unicode. state = ri.state + if isinstance(state, bytes): # Python 2 + state = str(state, "ascii") if state == "unstarted": return ConnectionStatus.unstarted()