ExternalVendorCode/tahoe-lafs
1
0
Fork 0
mirror of https://github.com/tahoe-lafs/tahoe-lafs.git synced 2024-12-23 23:02:25 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge pull request #877 from tahoe-lafs/3485.backported-configparser-for-py-2

Browse Source 
Backported configparser for Python 2

Fixes ticket:3485
This commit is contained in:
Itamar Turner-Trauring 2020-10-27 16:58:46 -04:00 committed by GitHub
commit 2f0a5d91cb
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 92 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

0
newsfragments/3485.minor Normal file
View File

4
nix/tahoe-lafs.nix
View File

@ -4,7 +4,7 @@
, setuptools, setuptoolsTrial, pyasn1, zope_interface
, service-identity, pyyaml, magic-wormhole, treq, appdirs
, beautifulsoup4, eliot, autobahn, cryptography, netifaces
, html5lib, pyutil, distro
, html5lib, pyutil, distro, configparser
}:
python.pkgs.buildPythonPackage rec {
version = "1.14.0.dev";
@ -46,7 +46,7 @@ python.pkgs.buildPythonPackage rec {
setuptoolsTrial pyasn1 zope_interface
service-identity pyyaml magic-wormhole treq
eliot autobahn cryptography netifaces setuptools
future pyutil distro
future pyutil distro configparser
];
checkInputs = with python.pkgs; [

3
setup.py
View File

@ -134,6 +134,9 @@ install_requires = [
# Linux distribution detection:
"distro >= 1.4.0",
# Backported configparser for Python 2:
"configparser ; python_version < '3.0'",
]
setup_requires = [

31
src/allmydata/client.py
View File

@ -2,10 +2,9 @@ import os, stat, time, weakref
from base64 import urlsafe_b64encode
from functools import partial
from errno import ENOENT, EPERM
try:
from ConfigParser import NoSectionError
except ImportError:
from configparser import NoSectionError
# On Python 2 this will be the backported package:
from configparser import NoSectionError
from foolscap.furl import (
decode_furl,
@ -35,8 +34,7 @@ from allmydata.util import (
hashutil, base32, pollmixin, log, idlib,
yamlutil, configutil,
)
from allmydata.util.encodingutil import (get_filesystem_encoding,
from_utf8_or_none)
from allmydata.util.encodingutil import get_filesystem_encoding
from allmydata.util.abbreviate import parse_abbreviated_size
from allmydata.util.time_format import parse_duration, parse_date
from allmydata.util.i2p_provider import create as create_i2p_provider
@ -719,6 +717,9 @@ class _Client(node.Node, pollmixin.PollMixin):
def init_stats_provider(self):
gatherer_furl = self.config.get_config("client", "stats_gatherer.furl", None)
if gatherer_furl:
# FURLs should be bytes:
gatherer_furl = gatherer_furl.encode("utf-8")
self.stats_provider = StatsProvider(self, gatherer_furl)
self.stats_provider.setServiceParent(self)
self.stats_provider.register_producer(self)
@ -806,7 +807,7 @@ class _Client(node.Node, pollmixin.PollMixin):
config_storedir = self.get_config(
"storage", "storage_dir", self.STOREDIR,
).decode('utf-8')
)
storedir = self.config.get_config_path(config_storedir)
data = self.config.get_config("storage", "reserved_space", None)
@ -935,6 +936,10 @@ class _Client(node.Node, pollmixin.PollMixin):
if helper_furl in ("None", ""):
helper_furl = None
# FURLs need to be bytes:
if helper_furl is not None:
helper_furl = helper_furl.encode("utf-8")
DEP = self.encoding_params
DEP["k"] = int(self.config.get_config("client", "shares.needed", DEP["k"]))
DEP["n"] = int(self.config.get_config("client", "shares.total", DEP["n"]))
@ -1043,15 +1048,14 @@ class _Client(node.Node, pollmixin.PollMixin):
from allmydata.webish import WebishServer
nodeurl_path = self.config.get_config_path("node.url")
staticdir_config = self.config.get_config("node", "web.static", "public_html").decode("utf-8")
staticdir_config = self.config.get_config("node", "web.static", "public_html")
staticdir = self.config.get_config_path(staticdir_config)
ws = WebishServer(self, webport, nodeurl_path, staticdir)
ws.setServiceParent(self)
def init_ftp_server(self):
if self.config.get_config("ftpd", "enabled", False, boolean=True):
accountfile = from_utf8_or_none(
self.config.get_config("ftpd", "accounts.file", None))
accountfile = self.config.get_config("ftpd", "accounts.file", None)
if accountfile:
accountfile = self.config.get_config_path(accountfile)
accounturl = self.config.get_config("ftpd", "accounts.url", None)
@ -1063,14 +1067,13 @@ class _Client(node.Node, pollmixin.PollMixin):
def init_sftp_server(self):
if self.config.get_config("sftpd", "enabled", False, boolean=True):
accountfile = from_utf8_or_none(
self.config.get_config("sftpd", "accounts.file", None))
accountfile = self.config.get_config("sftpd", "accounts.file", None)
if accountfile:
accountfile = self.config.get_config_path(accountfile)
accounturl = self.config.get_config("sftpd", "accounts.url", None)
sftp_portstr = self.config.get_config("sftpd", "port", "8022")
pubkey_file = from_utf8_or_none(self.config.get_config("sftpd", "host_pubkey_file"))
privkey_file = from_utf8_or_none(self.config.get_config("sftpd", "host_privkey_file"))
pubkey_file = self.config.get_config("sftpd", "host_pubkey_file")
privkey_file = self.config.get_config("sftpd", "host_privkey_file")
from allmydata.frontends import sftpd
s = sftpd.SFTPServer(self, accountfile, accounturl,

3
src/allmydata/frontends/ftpd.py
View File

@ -1,3 +1,4 @@
from six import ensure_str
from types import NoneType
@ -333,5 +334,7 @@ class FTPServer(service.MultiService):
raise NeedRootcapLookupScheme("must provide some translation")
f = ftp.FTPFactory(p)
# strports requires a native string.
ftp_portstr = ensure_str(ftp_portstr)
s = strports.service(ftp_portstr, f)
s.setServiceParent(self)

26
src/allmydata/node.py
View File

@ -3,21 +3,18 @@ This module contains classes and functions to implement and manage
a node for Tahoe-LAFS.
"""
from past.builtins import unicode
from six import ensure_str
import datetime
import os.path
import re
import types
import errno
from io import StringIO
import tempfile
from base64 import b32decode, b32encode
# Python 2 compatibility
from six.moves import configparser
from future.utils import PY2
if PY2:
from io import BytesIO as StringIO # noqa: F811
# On Python 2 this will be the backported package.
import configparser
from twisted.python import log as twlog
from twisted.application import service
@ -187,12 +184,13 @@ def read_config(basedir, portnumfile, generated_files=[], _valid_config=None):
# (try to) read the main config file
config_fname = os.path.join(basedir, "tahoe.cfg")
parser = configparser.SafeConfigParser()
try:
parser = configutil.get_config(config_fname)
except EnvironmentError as e:
if e.errno != errno.ENOENT:
raise
# The file is missing, just create empty ConfigParser.
parser = configutil.get_config_from_string(u"")
configutil.validate_config(config_fname, parser, _valid_config)
@ -209,9 +207,11 @@ def config_from_string(basedir, portnumfile, config_str, _valid_config=None):
if _valid_config is None:
_valid_config = _common_valid_config()
if isinstance(config_str, bytes):
config_str = config_str.decode("utf-8")
# load configuration from in-memory string
parser = configparser.SafeConfigParser()
parser.readfp(StringIO(config_str))
parser = configutil.get_config_from_string(config_str)
fname = "<in-memory>"
configutil.validate_config(fname, parser, _valid_config)
@ -639,6 +639,10 @@ def _tub_portlocation(config):
new_locations.append(loc)
location = ",".join(new_locations)
# Lacking this, Python 2 blows up in Foolscap when it is confused by a
# Unicode FURL.
location = location.encode("utf-8")
return tubport, location
@ -686,6 +690,9 @@ def create_main_tub(config, tub_options,
port_or_endpoint = tor_provider.get_listener()
else:
port_or_endpoint = port
# Foolscap requires native strings:
if isinstance(port_or_endpoint, (bytes, unicode)):
port_or_endpoint = ensure_str(port_or_endpoint)
tub.listenOn(port_or_endpoint)
tub.setLocation(location)
log.msg("Tub location set to %s" % (location,))
@ -839,6 +846,7 @@ class Node(service.MultiService):
lgfurl = self.config.get_config("node", "log_gatherer.furl", "")
if lgfurl:
# this is in addition to the contents of log-gatherer-furlfile
lgfurl = lgfurl.encode("utf-8")
self.log_tub.setOption("log-gatherer-furl", lgfurl)
self.log_tub.setOption("log-gatherer-furlfile",
self.config.get_config_path("log_gatherer.furl"))

4
src/allmydata/scripts/common.py
View File

@ -8,7 +8,9 @@ from os.path import join
from future.utils import PY2
if PY2:
from future.builtins import str # noqa: F401
from six.moves.configparser import NoSectionError
# On Python 2 this will be the backported package:
from configparser import NoSectionError
from twisted.python import usage

10
src/allmydata/storage_client.py
View File

@ -31,12 +31,10 @@ the foolscap-based server implemented in src/allmydata/storage/*.py .
from past.builtins import unicode
import re, time, hashlib
try:
from ConfigParser import (
NoSectionError,
)
except ImportError:
from configparser import NoSectionError
# On Python 2 this will be the backport.
from configparser import NoSectionError
import attr
from zope.interface import (
Attribute,

9
src/allmydata/test/test_configutil.py
View File

@ -154,3 +154,12 @@ enabled = false
self.dynamic_valid_config,
)
self.assertIn("section [node] contains unknown option 'invalid'", str(e))
def test_duplicate_sections(self):
"""
Duplicate section names are merged.
"""
fname = self.create_tahoe_cfg('[node]\na = foo\n[node]\n b = bar\n')
config = configutil.get_config(fname)
self.assertEqual(config.get("node", "a"), "foo")
self.assertEqual(config.get("node", "b"), "bar")

6
src/allmydata/test/test_connections.py
View File

@ -168,7 +168,11 @@ class Tor(unittest.TestCase):
tor_provider = create_tor_provider(reactor, config)
tor_provider.get_tor_handler()
self.assertIn(
"invalid literal for int() with base 10: 'kumquat'",
"invalid literal for int()",
str(ctx.exception)
)
self.assertIn(
"kumquat",
str(ctx.exception)
)

2
src/allmydata/test/test_node.py
View File

@ -151,7 +151,7 @@ class TestCase(testutil.SignalMixin, unittest.TestCase):
f.close()
config = read_config(basedir, "")
self.failUnlessEqual(config.get_config("node", "nickname").decode('utf-8'),
self.failUnlessEqual(config.get_config("node", "nickname"),
u"\u2621")
def test_tahoe_cfg_hash_in_name(self):

46
src/allmydata/util/configutil.py
View File

@ -1,7 +1,7 @@
"""
Read/write config files.
Configuration is returned as native strings.
Configuration is returned as Unicode strings.
Ported to Python 3.
"""
@ -12,17 +12,11 @@ from __future__ import unicode_literals
from future.utils import PY2
if PY2:
# We don't do open(), because we want files to read/write native strs when
# we do "r" or "w".
from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
if PY2:
# In theory on Python 2 configparser also works, but then code gets the
# wrong exceptions and they don't get handled. So just use native parser
# for now.
from ConfigParser import SafeConfigParser
else:
from configparser import SafeConfigParser
# On Python 2 we use the backport package; that means we always get unicode
# out.
from configparser import ConfigParser
import attr
@ -36,19 +30,27 @@ class UnknownConfigError(Exception):
def get_config(tahoe_cfg):
"""Load the config, returning a SafeConfigParser.
"""Load the config, returning a ConfigParser.
Configuration is returned as native strings.
Configuration is returned as Unicode strings.
"""
config = SafeConfigParser()
with open(tahoe_cfg, "r") as f:
# On Python 2, where we read in bytes, skip any initial Byte Order
# Mark. Since this is an ordinary file, we don't need to handle
# incomplete reads, and can assume seekability.
if PY2 and f.read(3) != b'\xEF\xBB\xBF':
f.seek(0)
config.readfp(f)
return config
# Byte Order Mark is an optional garbage code point you sometimes get at
# the start of UTF-8 encoded files. Especially on Windows. Skip it by using
# utf-8-sig. https://en.wikipedia.org/wiki/Byte_order_mark
with open(tahoe_cfg, "r", encoding="utf-8-sig") as f:
cfg_string = f.read()
return get_config_from_string(cfg_string)
def get_config_from_string(tahoe_cfg_string):
"""Load the config from a string, return the ConfigParser.
Configuration is returned as Unicode strings.
"""
parser = ConfigParser(strict=False)
parser.read_string(tahoe_cfg_string)
return parser
def set_config(config, section, option, value):
if not config.has_section(section):

4
src/allmydata/webish.py
View File

@ -1,3 +1,5 @@
from six import ensure_str
import re, time
from functools import (
@ -186,6 +188,8 @@ class WebishServer(service.MultiService):
self.root.putChild("static", static.File(staticdir))
if re.search(r'^\d', webport):
webport = "tcp:"+webport # twisted warns about bare "0" or "3456"
# strports must be native strings.
webport = ensure_str(webport)
s = strports.service(webport, self.site)
s.setServiceParent(self)