Merge pull request #772 from tahoe-lafs/3377.configutil-connection_status-python-3

Port configutil and connection_status to Python 3

Fixes ticket:3377

src/allmydata/interfaces.py

@@ -1,4 +1,3 @@
-
 from past.builtins import long
 
 from zope.interface import Interface, Attribute

src/allmydata/test/cli/test_create.py

@@ -6,6 +6,8 @@ from twisted.python import usage
 from allmydata.util import configutil
 from ..common_util import run_cli, parse_cli
 from ...scripts import create_node
+from ... import client
+
 
 def read_config(basedir):
     tahoe_cfg = os.path.join(basedir, "tahoe.cfg")
@@ -33,6 +35,31 @@ class Config(unittest.TestCase):
             e = self.assertRaises(usage.UsageError, parse_cli, verb, *args)
             self.assertIn("option %s not recognized" % (option,), str(e))
 
+    def test_create_client_config(self):
+        d = self.mktemp()
+        os.mkdir(d)
+        fname = os.path.join(d, 'tahoe.cfg')
+
+        with open(fname, 'w') as f:
+            opts = {"nickname": "nick",
+                    "webport": "tcp:3456",
+                    "hide-ip": False,
+                    "listen": "none",
+                    "shares-needed": "1",
+                    "shares-happy": "1",
+                    "shares-total": "1",
+                    }
+            create_node.write_node_config(f, opts)
+            create_node.write_client_config(f, opts)
+
+        config = configutil.get_config(fname)
+        # should succeed, no exceptions
+        configutil.validate_config(
+            fname,
+            config,
+            client._valid_config(),
+        )
+
     @defer.inlineCallbacks
     def test_client(self):
         basedir = self.mktemp()

src/allmydata/test/test_configutil.py

@@ -1,14 +1,26 @@
+"""
+Tests for allmydata.util.configutil.
+
+Ported to Python 3.
+"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from future.utils import PY2
+if PY2:
+    # Omitted dict, cause worried about interactions.
+    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, list, object, range, str, max, min  # noqa: F401
+
 import os.path
 
 from twisted.trial import unittest
 
 from allmydata.util import configutil
-from allmydata.test.no_network import GridTestMixin
-from ..scripts import create_node
-from .. import client
 
 
-class ConfigUtilTests(GridTestMixin, unittest.TestCase):
+class ConfigUtilTests(unittest.TestCase):
     def setUp(self):
         super(ConfigUtilTests, self).setUp()
         self.static_valid_config = configutil.ValidConfiguration(
@@ -20,10 +32,22 @@ class ConfigUtilTests(GridTestMixin, unittest.TestCase):
             lambda section_name, item_name: (section_name, item_name) == ("node", "valid"),
         )
 
+    def create_tahoe_cfg(self, cfg):
+        d = self.mktemp()
+        os.mkdir(d)
+        fname = os.path.join(d, 'tahoe.cfg')
+        with open(fname, "w") as f:
+            f.write(cfg)
+        return fname
+
     def test_config_utils(self):
-        self.basedir = "cli/ConfigUtilTests/test-config-utils"
+        tahoe_cfg = self.create_tahoe_cfg("""\
-        self.set_up_grid(oneshare=True)
+[node]
-        tahoe_cfg = os.path.join(self.get_clientdir(i=0), "tahoe.cfg")
+nickname = client-0
+web.port = adopt-socket:fd=5
+[storage]
+enabled = false
+""")
 
         # test that at least one option was read correctly
         config = configutil.get_config(tahoe_cfg)
@@ -45,12 +69,7 @@ class ConfigUtilTests(GridTestMixin, unittest.TestCase):
         self.failUnlessEqual(config.get("node", "descriptor"), descriptor)
 
     def test_config_validation_success(self):
-        d = self.mktemp()
+        fname = self.create_tahoe_cfg('[node]\nvalid = foo\n')
-        os.mkdir(d)
-        fname = os.path.join(d, 'tahoe.cfg')
-
-        with open(fname, 'w') as f:
-            f.write('[node]\nvalid = foo\n')
 
         config = configutil.get_config(fname)
         # should succeed, no exceptions
@@ -66,12 +85,7 @@ class ConfigUtilTests(GridTestMixin, unittest.TestCase):
         validation but are matched by the dynamic validation is considered
         valid.
         """
-        d = self.mktemp()
+        fname = self.create_tahoe_cfg('[node]\nvalid = foo\n')
-        os.mkdir(d)
-        fname = os.path.join(d, 'tahoe.cfg')
-
-        with open(fname, 'w') as f:
-            f.write('[node]\nvalid = foo\n')
 
         config = configutil.get_config(fname)
         # should succeed, no exceptions
@@ -82,12 +96,7 @@ class ConfigUtilTests(GridTestMixin, unittest.TestCase):
         )
 
     def test_config_validation_invalid_item(self):
-        d = self.mktemp()
+        fname = self.create_tahoe_cfg('[node]\nvalid = foo\ninvalid = foo\n')
-        os.mkdir(d)
-        fname = os.path.join(d, 'tahoe.cfg')
-
-        with open(fname, 'w') as f:
-            f.write('[node]\nvalid = foo\ninvalid = foo\n')
 
         config = configutil.get_config(fname)
         e = self.assertRaises(
@@ -103,12 +112,7 @@ class ConfigUtilTests(GridTestMixin, unittest.TestCase):
         A configuration with a section that is matched by neither the static nor
         dynamic validators is rejected.
         """
-        d = self.mktemp()
+        fname = self.create_tahoe_cfg('[node]\nvalid = foo\n[invalid]\n')
-        os.mkdir(d)
-        fname = os.path.join(d, 'tahoe.cfg')
-
-        with open(fname, 'w') as f:
-            f.write('[node]\nvalid = foo\n[invalid]\n')
 
         config = configutil.get_config(fname)
         e = self.assertRaises(
@@ -124,12 +128,7 @@ class ConfigUtilTests(GridTestMixin, unittest.TestCase):
         A configuration with a section that is matched by neither the static nor
         dynamic validators is rejected.
         """
-        d = self.mktemp()
+        fname = self.create_tahoe_cfg('[node]\nvalid = foo\n[invalid]\n')
-        os.mkdir(d)
-        fname = os.path.join(d, 'tahoe.cfg')
-
-        with open(fname, 'w') as f:
-            f.write('[node]\nvalid = foo\n[invalid]\n')
 
         config = configutil.get_config(fname)
         e = self.assertRaises(
@@ -145,12 +144,7 @@ class ConfigUtilTests(GridTestMixin, unittest.TestCase):
         A configuration with a section, item pair that is matched by neither the
         static nor dynamic validators is rejected.
         """
-        d = self.mktemp()
+        fname = self.create_tahoe_cfg('[node]\nvalid = foo\ninvalid = foo\n')
-        os.mkdir(d)
-        fname = os.path.join(d, 'tahoe.cfg')
-
-        with open(fname, 'w') as f:
-            f.write('[node]\nvalid = foo\ninvalid = foo\n')
 
         config = configutil.get_config(fname)
         e = self.assertRaises(
@@ -160,28 +154,3 @@ class ConfigUtilTests(GridTestMixin, unittest.TestCase):
             self.dynamic_valid_config,
         )
         self.assertIn("section [node] contains unknown option 'invalid'", str(e))
-
-    def test_create_client_config(self):
-        d = self.mktemp()
-        os.mkdir(d)
-        fname = os.path.join(d, 'tahoe.cfg')
-
-        with open(fname, 'w') as f:
-            opts = {"nickname": "nick",
-                    "webport": "tcp:3456",
-                    "hide-ip": False,
-                    "listen": "none",
-                    "shares-needed": "1",
-                    "shares-happy": "1",
-                    "shares-total": "1",
-                    }
-            create_node.write_node_config(f, opts)
-            create_node.write_client_config(f, opts)
-
-        config = configutil.get_config(fname)
-        # should succeed, no exceptions
-        configutil.validate_config(
-            fname,
-            config,
-            client._valid_config(),
-        )

src/allmydata/test/test_connection_status.py

@@ -0,0 +1,122 @@
+"""
+Tests for allmydata.util.connection_status.
+
+Port to Python 3.
+"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from future.utils import PY2
+if PY2:
+    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
+
+import mock
+
+from twisted.trial import unittest
+
+from ..util import connection_status
+
+class Status(unittest.TestCase):
+    def test_hint_statuses(self):
+        ncs = connection_status._hint_statuses(["h2","h1"],
+                                               {"h1": "hand1", "h4": "hand4"},
+                                               {"h1": "st1", "h2": "st2",
+                                                "h3": "st3"})
+        self.assertEqual(ncs, {"h1 via hand1": "st1",
+                               "h2": "st2"})
+
+    def test_reconnector_connected(self):
+        ci = mock.Mock()
+        ci.connectorStatuses = {"h1": "st1"}
+        ci.connectionHandlers = {"h1": "hand1"}
+        ci.winningHint = "h1"
+        ci.establishedAt = 120
+        ri = mock.Mock()
+        ri.state = "connected"
+        ri.connectionInfo = ci
+        rc = mock.Mock
+        rc.getReconnectionInfo = mock.Mock(return_value=ri)
+        cs = connection_status.from_foolscap_reconnector(rc, 123)
+        self.assertEqual(cs.connected, True)
+        self.assertEqual(cs.summary, "Connected to h1 via hand1")
+        self.assertEqual(cs.non_connected_statuses, {})
+        self.assertEqual(cs.last_connection_time, 120)
+        self.assertEqual(cs.last_received_time, 123)
+
+    def test_reconnector_connected_others(self):
+        ci = mock.Mock()
+        ci.connectorStatuses = {"h1": "st1", "h2": "st2"}
+        ci.connectionHandlers = {"h1": "hand1"}
+        ci.winningHint = "h1"
+        ci.establishedAt = 120
+        ri = mock.Mock()
+        ri.state = "connected"
+        ri.connectionInfo = ci
+        rc = mock.Mock
+        rc.getReconnectionInfo = mock.Mock(return_value=ri)
+        cs = connection_status.from_foolscap_reconnector(rc, 123)
+        self.assertEqual(cs.connected, True)
+        self.assertEqual(cs.summary, "Connected to h1 via hand1")
+        self.assertEqual(cs.non_connected_statuses, {"h2": "st2"})
+        self.assertEqual(cs.last_connection_time, 120)
+        self.assertEqual(cs.last_received_time, 123)
+
+    def test_reconnector_connected_listener(self):
+        ci = mock.Mock()
+        ci.connectorStatuses = {"h1": "st1", "h2": "st2"}
+        ci.connectionHandlers = {"h1": "hand1"}
+        ci.listenerStatus = ("listener1", "successful")
+        ci.winningHint = None
+        ci.establishedAt = 120
+        ri = mock.Mock()
+        ri.state = "connected"
+        ri.connectionInfo = ci
+        rc = mock.Mock
+        rc.getReconnectionInfo = mock.Mock(return_value=ri)
+        cs = connection_status.from_foolscap_reconnector(rc, 123)
+        self.assertEqual(cs.connected, True)
+        self.assertEqual(cs.summary, "Connected via listener (listener1)")
+        self.assertEqual(cs.non_connected_statuses,
+                         {"h1 via hand1": "st1", "h2": "st2"})
+        self.assertEqual(cs.last_connection_time, 120)
+        self.assertEqual(cs.last_received_time, 123)
+
+    def test_reconnector_connecting(self):
+        ci = mock.Mock()
+        ci.connectorStatuses = {"h1": "st1", "h2": "st2"}
+        ci.connectionHandlers = {"h1": "hand1"}
+        ri = mock.Mock()
+        ri.state = "connecting"
+        ri.connectionInfo = ci
+        rc = mock.Mock
+        rc.getReconnectionInfo = mock.Mock(return_value=ri)
+        cs = connection_status.from_foolscap_reconnector(rc, 123)
+        self.assertEqual(cs.connected, False)
+        self.assertEqual(cs.summary, "Trying to connect")
+        self.assertEqual(cs.non_connected_statuses,
+                         {"h1 via hand1": "st1", "h2": "st2"})
+        self.assertEqual(cs.last_connection_time, None)
+        self.assertEqual(cs.last_received_time, 123)
+
+    def test_reconnector_waiting(self):
+        ci = mock.Mock()
+        ci.connectorStatuses = {"h1": "st1", "h2": "st2"}
+        ci.connectionHandlers = {"h1": "hand1"}
+        ri = mock.Mock()
+        ri.state = "waiting"
+        ri.lastAttempt = 10
+        ri.nextAttempt = 20
+        ri.connectionInfo = ci
+        rc = mock.Mock
+        rc.getReconnectionInfo = mock.Mock(return_value=ri)
+        with mock.patch("time.time", return_value=12):
+            cs = connection_status.from_foolscap_reconnector(rc, 5)
+        self.assertEqual(cs.connected, False)
+        self.assertEqual(cs.summary,
+                         "Reconnecting in 8 seconds (last attempt 2s ago)")
+        self.assertEqual(cs.non_connected_statuses,
+                         {"h1 via hand1": "st1", "h2": "st2"})
+        self.assertEqual(cs.last_connection_time, None)
+        self.assertEqual(cs.last_received_time, 5)

src/allmydata/test/test_connections.py

@@ -7,7 +7,6 @@ from foolscap.connections import tcp
 from ..node import PrivacyError, config_from_string
 from ..node import create_connection_handlers
 from ..node import create_main_tub, _tub_portlocation
-from ..util import connection_status
 from ..util.i2p_provider import create as create_i2p_provider
 from ..util.tor_provider import create as create_tor_provider
 
@@ -463,106 +462,3 @@ class Privacy(unittest.TestCase):
             str(ctx.exception),
             "tub.location includes tcp: hint",
         )
-
-class Status(unittest.TestCase):
-    def test_hint_statuses(self):
-        ncs = connection_status._hint_statuses(["h2","h1"],
-                                               {"h1": "hand1", "h4": "hand4"},
-                                               {"h1": "st1", "h2": "st2",
-                                                "h3": "st3"})
-        self.assertEqual(ncs, {"h1 via hand1": "st1",
-                               "h2": "st2"})
-
-    def test_reconnector_connected(self):
-        ci = mock.Mock()
-        ci.connectorStatuses = {"h1": "st1"}
-        ci.connectionHandlers = {"h1": "hand1"}
-        ci.winningHint = "h1"
-        ci.establishedAt = 120
-        ri = mock.Mock()
-        ri.state = "connected"
-        ri.connectionInfo = ci
-        rc = mock.Mock
-        rc.getReconnectionInfo = mock.Mock(return_value=ri)
-        cs = connection_status.from_foolscap_reconnector(rc, 123)
-        self.assertEqual(cs.connected, True)
-        self.assertEqual(cs.summary, "Connected to h1 via hand1")
-        self.assertEqual(cs.non_connected_statuses, {})
-        self.assertEqual(cs.last_connection_time, 120)
-        self.assertEqual(cs.last_received_time, 123)
-
-    def test_reconnector_connected_others(self):
-        ci = mock.Mock()
-        ci.connectorStatuses = {"h1": "st1", "h2": "st2"}
-        ci.connectionHandlers = {"h1": "hand1"}
-        ci.winningHint = "h1"
-        ci.establishedAt = 120
-        ri = mock.Mock()
-        ri.state = "connected"
-        ri.connectionInfo = ci
-        rc = mock.Mock
-        rc.getReconnectionInfo = mock.Mock(return_value=ri)
-        cs = connection_status.from_foolscap_reconnector(rc, 123)
-        self.assertEqual(cs.connected, True)
-        self.assertEqual(cs.summary, "Connected to h1 via hand1")
-        self.assertEqual(cs.non_connected_statuses, {"h2": "st2"})
-        self.assertEqual(cs.last_connection_time, 120)
-        self.assertEqual(cs.last_received_time, 123)
-
-    def test_reconnector_connected_listener(self):
-        ci = mock.Mock()
-        ci.connectorStatuses = {"h1": "st1", "h2": "st2"}
-        ci.connectionHandlers = {"h1": "hand1"}
-        ci.listenerStatus = ("listener1", "successful")
-        ci.winningHint = None
-        ci.establishedAt = 120
-        ri = mock.Mock()
-        ri.state = "connected"
-        ri.connectionInfo = ci
-        rc = mock.Mock
-        rc.getReconnectionInfo = mock.Mock(return_value=ri)
-        cs = connection_status.from_foolscap_reconnector(rc, 123)
-        self.assertEqual(cs.connected, True)
-        self.assertEqual(cs.summary, "Connected via listener (listener1)")
-        self.assertEqual(cs.non_connected_statuses,
-                         {"h1 via hand1": "st1", "h2": "st2"})
-        self.assertEqual(cs.last_connection_time, 120)
-        self.assertEqual(cs.last_received_time, 123)
-
-    def test_reconnector_connecting(self):
-        ci = mock.Mock()
-        ci.connectorStatuses = {"h1": "st1", "h2": "st2"}
-        ci.connectionHandlers = {"h1": "hand1"}
-        ri = mock.Mock()
-        ri.state = "connecting"
-        ri.connectionInfo = ci
-        rc = mock.Mock
-        rc.getReconnectionInfo = mock.Mock(return_value=ri)
-        cs = connection_status.from_foolscap_reconnector(rc, 123)
-        self.assertEqual(cs.connected, False)
-        self.assertEqual(cs.summary, "Trying to connect")
-        self.assertEqual(cs.non_connected_statuses,
-                         {"h1 via hand1": "st1", "h2": "st2"})
-        self.assertEqual(cs.last_connection_time, None)
-        self.assertEqual(cs.last_received_time, 123)
-
-    def test_reconnector_waiting(self):
-        ci = mock.Mock()
-        ci.connectorStatuses = {"h1": "st1", "h2": "st2"}
-        ci.connectionHandlers = {"h1": "hand1"}
-        ri = mock.Mock()
-        ri.state = "waiting"
-        ri.lastAttempt = 10
-        ri.nextAttempt = 20
-        ri.connectionInfo = ci
-        rc = mock.Mock
-        rc.getReconnectionInfo = mock.Mock(return_value=ri)
-        with mock.patch("time.time", return_value=12):
-            cs = connection_status.from_foolscap_reconnector(rc, 5)
-        self.assertEqual(cs.connected, False)
-        self.assertEqual(cs.summary,
-                         "Reconnecting in 8 seconds (last attempt 2s ago)")
-        self.assertEqual(cs.non_connected_statuses,
-                         {"h1 via hand1": "st1", "h2": "st2"})
-        self.assertEqual(cs.last_connection_time, None)
-        self.assertEqual(cs.last_received_time, 5)

src/allmydata/util/_python3.py

@@ -41,6 +41,8 @@ PORTED_MODULES = [
     "allmydata.util.assertutil",
     "allmydata.util.base32",
     "allmydata.util.base62",
+    "allmydata.util.configutil",
+    "allmydata.util.connection_status",
     "allmydata.util.deferredutil",
     "allmydata.util.fileutil",
     "allmydata.util.dictutil",
@@ -66,6 +68,8 @@ PORTED_TEST_MODULES = [
     "allmydata.test.test_abbreviate",
     "allmydata.test.test_base32",
     "allmydata.test.test_base62",
+    "allmydata.test.test_configutil",
+    "allmydata.test.test_connection_status",
     "allmydata.test.test_crawler",
     "allmydata.test.test_crypto",
     "allmydata.test.test_deferredutil",

src/allmydata/util/configutil.py

@@ -1,8 +1,32 @@
+"""
+Read/write config files.
 
-from ConfigParser import SafeConfigParser
+Configuration is returned as native strings.
+
+Ported to Python 3.
+"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from future.utils import PY2
+if PY2:
+    # We don't do open(), because we want files to read/write native strs when
+    # we do "r" or "w".
+    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
+
+if PY2:
+    # In theory on Python 2 configparser also works, but then code gets the
+    # wrong exceptions and they don't get handled. So just use native parser
+    # for now.
+    from ConfigParser import SafeConfigParser
+else:
+    from configparser import SafeConfigParser
 
 import attr
 
+
 class UnknownConfigError(Exception):
     """
     An unknown config item was found.
@@ -12,11 +36,16 @@ class UnknownConfigError(Exception):
 
 
 def get_config(tahoe_cfg):
+    """Load the config, returning a SafeConfigParser.
+
+    Configuration is returned as native strings.
+    """
     config = SafeConfigParser()
-    with open(tahoe_cfg, "rb") as f:
+    with open(tahoe_cfg, "r") as f:
-        # Skip any initial Byte Order Mark. Since this is an ordinary file, we
+        # On Python 2, where we read in bytes, skip any initial Byte Order
-        # don't need to handle incomplete reads, and can assume seekability.
+        # Mark. Since this is an ordinary file, we don't need to handle
-        if f.read(3) != '\xEF\xBB\xBF':
+        # incomplete reads, and can assume seekability.
+        if PY2 and f.read(3) != b'\xEF\xBB\xBF':
             f.seek(0)
         config.readfp(f)
     return config
@@ -28,7 +57,7 @@ def set_config(config, section, option, value):
     assert config.get(section, option) == value
 
 def write_config(tahoe_cfg, config):
-    with open(tahoe_cfg, "wb") as f:
+    with open(tahoe_cfg, "w") as f:
         config.write(f)
 
 def validate_config(fname, cfg, valid_config):

src/allmydata/util/connection_status.py

@@ -1,3 +1,18 @@
+"""
+Parse connection status from Foolscap.
+
+Ported to Python 3.
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from future.utils import PY2
+if PY2:
+    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
+
 import time
 from zope.interface import implementer
 from ..interfaces import IConnectionStatus
@@ -37,9 +52,12 @@ def _hint_statuses(which, handlers, statuses):
 
 def from_foolscap_reconnector(rc, last_received):
     ri = rc.getReconnectionInfo()
-    # See foolscap/reconnector.py, ReconnectionInfo, for details about
+    # See foolscap/reconnector.py, ReconnectionInfo, for details about possible
-    # possible states.
+    # states. The returned result is a native string, it seems, so convert to
+    # unicode.
     state = ri.state
+    if isinstance(state, bytes):  # Python 2
+        state = str(state, "ascii")
     if state == "unstarted":
         return ConnectionStatus.unstarted()