Merge branch '3485.backported-configparser-for-py-2' into 3479.test-node-python-3

nix/tahoe-lafs.nix

@@ -4,7 +4,7 @@
 , setuptools, setuptoolsTrial, pyasn1, zope_interface
 , service-identity, pyyaml, magic-wormhole, treq, appdirs
 , beautifulsoup4, eliot, autobahn, cryptography, netifaces
-, html5lib, pyutil, distro
+, html5lib, pyutil, distro, configparser
 }:
 python.pkgs.buildPythonPackage rec {
   version = "1.14.0.dev";
@@ -46,7 +46,7 @@ python.pkgs.buildPythonPackage rec {
     setuptoolsTrial pyasn1 zope_interface
     service-identity pyyaml magic-wormhole treq
     eliot autobahn cryptography netifaces setuptools
-    future pyutil distro
+    future pyutil distro configparser
   ];
 
   checkInputs = with python.pkgs; [

setup.py

@@ -134,6 +134,9 @@ install_requires = [
 
     # Linux distribution detection:
     "distro >= 1.4.0",
+
+    # Backported configparser for Python 2:
+    "configparser ; python_version < '3.0'",
 ]
 
 setup_requires = [

src/allmydata/client.py

@@ -2,10 +2,9 @@ import os, stat, time, weakref
 from base64 import urlsafe_b64encode
 from functools import partial
 from errno import ENOENT, EPERM
-try:
+
-    from ConfigParser import NoSectionError
+# On Python 2 this will be the backported package:
-except ImportError:
+from configparser import NoSectionError
-    from configparser import NoSectionError
 
 from foolscap.furl import (
     decode_furl,
@@ -35,8 +34,7 @@ from allmydata.util import (
     hashutil, base32, pollmixin, log, idlib,
     yamlutil, configutil,
 )
-from allmydata.util.encodingutil import (get_filesystem_encoding,
+from allmydata.util.encodingutil import get_filesystem_encoding
-                                         from_utf8_or_none)
 from allmydata.util.abbreviate import parse_abbreviated_size
 from allmydata.util.time_format import parse_duration, parse_date
 from allmydata.util.i2p_provider import create as create_i2p_provider
@@ -719,6 +717,9 @@ class _Client(node.Node, pollmixin.PollMixin):
 
     def init_stats_provider(self):
         gatherer_furl = self.config.get_config("client", "stats_gatherer.furl", None)
+        if gatherer_furl:
+            # FURLs should be bytes:
+            gatherer_furl = gatherer_furl.encode("utf-8")
         self.stats_provider = StatsProvider(self, gatherer_furl)
         self.stats_provider.setServiceParent(self)
         self.stats_provider.register_producer(self)
@@ -935,6 +936,10 @@ class _Client(node.Node, pollmixin.PollMixin):
         if helper_furl in ("None", ""):
             helper_furl = None
 
+        # FURLs need to be bytes:
+        if helper_furl is not None:
+            helper_furl = helper_furl.encode("utf-8")
+
         DEP = self.encoding_params
         DEP["k"] = int(self.config.get_config("client", "shares.needed", DEP["k"]))
         DEP["n"] = int(self.config.get_config("client", "shares.total", DEP["n"]))
@@ -1043,15 +1048,14 @@ class _Client(node.Node, pollmixin.PollMixin):
 
         from allmydata.webish import WebishServer
         nodeurl_path = self.config.get_config_path("node.url")
-        staticdir_config = self.config.get_config("node", "web.static", "public_html").decode("utf-8")
+        staticdir_config = self.config.get_config("node", "web.static", "public_html")
         staticdir = self.config.get_config_path(staticdir_config)
         ws = WebishServer(self, webport, nodeurl_path, staticdir)
         ws.setServiceParent(self)
 
     def init_ftp_server(self):
         if self.config.get_config("ftpd", "enabled", False, boolean=True):
-            accountfile = from_utf8_or_none(
+            accountfile = self.config.get_config("ftpd", "accounts.file", None)
-                self.config.get_config("ftpd", "accounts.file", None))
             if accountfile:
                 accountfile = self.config.get_config_path(accountfile)
             accounturl = self.config.get_config("ftpd", "accounts.url", None)
@@ -1063,14 +1067,13 @@ class _Client(node.Node, pollmixin.PollMixin):
 
     def init_sftp_server(self):
         if self.config.get_config("sftpd", "enabled", False, boolean=True):
-            accountfile = from_utf8_or_none(
+            accountfile = self.config.get_config("sftpd", "accounts.file", None)
-                self.config.get_config("sftpd", "accounts.file", None))
             if accountfile:
                 accountfile = self.config.get_config_path(accountfile)
             accounturl = self.config.get_config("sftpd", "accounts.url", None)
             sftp_portstr = self.config.get_config("sftpd", "port", "8022")
-            pubkey_file = from_utf8_or_none(self.config.get_config("sftpd", "host_pubkey_file"))
+            pubkey_file = self.config.get_config("sftpd", "host_pubkey_file")
-            privkey_file = from_utf8_or_none(self.config.get_config("sftpd", "host_privkey_file"))
+            privkey_file = self.config.get_config("sftpd", "host_privkey_file")
 
             from allmydata.frontends import sftpd
             s = sftpd.SFTPServer(self, accountfile, accounturl,

src/allmydata/frontends/ftpd.py

@@ -1,3 +1,4 @@
+from six import ensure_str
 
 from types import NoneType
 
@@ -333,5 +334,7 @@ class FTPServer(service.MultiService):
             raise NeedRootcapLookupScheme("must provide some translation")
 
         f = ftp.FTPFactory(p)
+        # strports requires a native string.
+        ftp_portstr = ensure_str(ftp_portstr)
         s = strports.service(ftp_portstr, f)
         s.setServiceParent(self)

src/allmydata/node.py

@@ -3,19 +3,18 @@ This module contains classes and functions to implement and manage
 a node for Tahoe-LAFS.
 """
 from past.builtins import unicode
+from six import ensure_str
 
 import datetime
 import os.path
 import re
 import types
 import errno
-from io import StringIO, BytesIO
 import tempfile
 from base64 import b32decode, b32encode
 
-# Python 2 compatibility
+# On Python 2 this will be the backported package.
-from six.moves import configparser
+import configparser
-from future.utils import PY2
 
 from twisted.python import log as twlog
 from twisted.application import service
@@ -185,12 +184,13 @@ def read_config(basedir, portnumfile, generated_files=[], _valid_config=None):
 
     # (try to) read the main config file
     config_fname = os.path.join(basedir, "tahoe.cfg")
-    parser = configparser.SafeConfigParser()
     try:
         parser = configutil.get_config(config_fname)
     except EnvironmentError as e:
         if e.errno != errno.ENOENT:
             raise
+        # The file is missing, just create empty ConfigParser.
+        parser = configutil.get_config_from_string(u"")
 
     configutil.validate_config(config_fname, parser, _valid_config)
 
@@ -207,12 +207,11 @@ def config_from_string(basedir, portnumfile, config_str, _valid_config=None):
     if _valid_config is None:
         _valid_config = _common_valid_config()
 
+    if isinstance(config_str, bytes):
+        config_str = config_str.decode("utf-8")
+
     # load configuration from in-memory string
-    parser = configparser.SafeConfigParser()
+    parser = configutil.get_config_from_string(config_str)
-    if PY2:
-        parser.readfp(BytesIO(config_str.encode("utf-8")))
-    else:
-        parser.readfp(StringIO(config_str))
 
     fname = "<in-memory>"
     configutil.validate_config(fname, parser, _valid_config)
@@ -647,6 +646,10 @@ def _tub_portlocation(config):
             new_locations.append(loc)
     location = ",".join(new_locations)
 
+    # Lacking this, Python 2 blows up in Foolscap when it is confused by a
+    # Unicode FURL.
+    location = location.encode("utf-8")
+
     return tubport, location
 
 
@@ -694,9 +697,9 @@ def create_main_tub(config, tub_options,
                 port_or_endpoint = tor_provider.get_listener()
             else:
                 port_or_endpoint = port
-            if PY2 and isinstance(port_or_endpoint, unicode):
+            # Foolscap requires native strings:
-                # Foolscap requires native string
+            if isinstance(port_or_endpoint, (bytes, unicode)):
-                port_or_endpoint = port_or_endpoint.encode("utf-8")
+                port_or_endpoint = ensure_str(port_or_endpoint)
             tub.listenOn(port_or_endpoint)
         tub.setLocation(location)
         log.msg("Tub location set to %s" % (location,))
@@ -850,6 +853,7 @@ class Node(service.MultiService):
         lgfurl = self.config.get_config("node", "log_gatherer.furl", "")
         if lgfurl:
             # this is in addition to the contents of log-gatherer-furlfile
+            lgfurl = lgfurl.encode("utf-8")
             self.log_tub.setOption("log-gatherer-furl", lgfurl)
         self.log_tub.setOption("log-gatherer-furlfile",
                                self.config.get_config_path("log_gatherer.furl"))

src/allmydata/scripts/common.py

@@ -8,7 +8,9 @@ from os.path import join
 from future.utils import PY2
 if PY2:
     from future.builtins import str  # noqa: F401
-from six.moves.configparser import NoSectionError
+
+# On Python 2 this will be the backported package:
+from configparser import NoSectionError
 
 from twisted.python import usage
 

src/allmydata/storage_client.py

@@ -31,12 +31,10 @@ the foolscap-based server implemented in src/allmydata/storage/*.py .
 from past.builtins import unicode
 
 import re, time, hashlib
-try:
+
-    from ConfigParser import (
+# On Python 2 this will be the backport.
-        NoSectionError,
+from configparser import NoSectionError
-    )
+
-except ImportError:
-    from configparser import NoSectionError
 import attr
 from zope.interface import (
     Attribute,

src/allmydata/test/test_configutil.py

@@ -154,3 +154,12 @@ enabled = false
             self.dynamic_valid_config,
         )
         self.assertIn("section [node] contains unknown option 'invalid'", str(e))
+
+    def test_duplicate_sections(self):
+        """
+        Duplicate section names are merged.
+        """
+        fname = self.create_tahoe_cfg('[node]\na = foo\n[node]\n b = bar\n')
+        config = configutil.get_config(fname)
+        self.assertEqual(config.get("node", "a"), "foo")
+        self.assertEqual(config.get("node", "b"), "bar")

src/allmydata/test/test_connections.py

@@ -168,7 +168,11 @@ class Tor(unittest.TestCase):
             tor_provider = create_tor_provider(reactor, config)
             tor_provider.get_tor_handler()
         self.assertIn(
-            "invalid literal for int() with base 10: 'kumquat'",
+            "invalid literal for int()",
+            str(ctx.exception)
+        )
+        self.assertIn(
+            "kumquat",
             str(ctx.exception)
         )
 

src/allmydata/test/test_node.py

@@ -163,12 +163,8 @@ class TestCase(testutil.SignalMixin, unittest.TestCase):
         f.close()
 
         config = read_config(basedir, "")
-        # Config returns native strings:
-        expected_nick = u"\u2621"
-        if PY2:
-            expected_nick = expected_nick.encode("utf-8")
         self.failUnlessEqual(config.get_config("node", "nickname"),
-                             expected_nick)
+                             u"\u2621")
 
     def test_tahoe_cfg_hash_in_name(self):
         basedir = "test_node/test_cfg_hash_in_name"

src/allmydata/util/configutil.py

@@ -1,7 +1,7 @@
 """
 Read/write config files.
 
-Configuration is returned as native strings.
+Configuration is returned as Unicode strings.
 
 Ported to Python 3.
 """
@@ -12,17 +12,11 @@ from __future__ import unicode_literals
 
 from future.utils import PY2, PY3
 if PY2:
-    # We don't do open(), because we want files to read/write native strs when
+    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
-    # we do "r" or "w".
-    from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
-if PY2:
+# On Python 2 we use the backport package; that means we always get unicode
-    # In theory on Python 2 configparser also works, but then code gets the
+# out.
-    # wrong exceptions and they don't get handled. So just use native parser
+from configparser import ConfigParser
-    # for now.
-    from ConfigParser import SafeConfigParser
-else:
-    from configparser import SafeConfigParser
 
 import attr
 
@@ -36,20 +30,27 @@ class UnknownConfigError(Exception):
 
 
 def get_config(tahoe_cfg):
-    """Load the config, returning a SafeConfigParser.
+    """Load the config, returning a ConfigParser.
 
-    Configuration is returned as native strings.
+    Configuration is returned as Unicode strings.
     """
-    config = SafeConfigParser()
+    # Byte Order Mark is an optional garbage code point you sometimes get at
-    with open(tahoe_cfg, "r") as f:
+    # the start of UTF-8 encoded files. Especially on Windows. Skip it by using
-        # Skip any initial Byte Order Mark. Since this is an ordinary file, we
+    # utf-8-sig. https://en.wikipedia.org/wiki/Byte_order_mark
-        # don't need to handle incomplete reads, and can assume seekability.
+    with open(tahoe_cfg, "r", encoding="utf-8-sig") as f:
-        if PY2 and f.read(3) != b'\xEF\xBB\xBF':
+        cfg_string = f.read()
-            f.seek(0)
+    return get_config_from_string(cfg_string)
-        if PY3 and f.read(1) != u"\uFEFF":
+
-            f.seek(0)
+
-        config.readfp(f)
+def get_config_from_string(tahoe_cfg_string):
-    return config
+    """Load the config from a string, return the ConfigParser.
+
+    Configuration is returned as Unicode strings.
+    """
+    parser = ConfigParser(strict=False)
+    parser.read_string(tahoe_cfg_string)
+    return parser
+
 
 def set_config(config, section, option, value):
     if not config.has_section(section):

src/allmydata/webish.py

@@ -1,3 +1,5 @@
+from six import ensure_str
+
 import re, time
 
 from functools import (
@@ -186,6 +188,8 @@ class WebishServer(service.MultiService):
             self.root.putChild("static", static.File(staticdir))
         if re.search(r'^\d', webport):
             webport = "tcp:"+webport # twisted warns about bare "0" or "3456"
+        # strports must be native strings.
+        webport = ensure_str(webport)
         s = strports.service(webport, self.site)
         s.setServiceParent(self)