ExternalVendorCode/tahoe-lafs
1
0
Fork 0
mirror of https://github.com/tahoe-lafs/tahoe-lafs.git synced 2024-12-21 13:57:51 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge pull request #900 from tahoe-lafs/3511.config-set-config

Browse Source 
Add `_Config.set_config`

Fixes: ticket:3511
This commit is contained in:
Jean-Paul Calderone 2020-12-01 12:04:26 -05:00 committed by GitHub
commit 8d6b49669b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 406 additions and 74 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
integration/util.py
View File

@ -6,6 +6,9 @@ from os.path import exists, join
from six.moves import StringIO
from functools import partial
from twisted.python.filepath import (
FilePath,
)
from twisted.internet.defer import Deferred, succeed
from twisted.internet.protocol import ProcessProtocol
from twisted.internet.error import ProcessExitedAlready, ProcessDone
@ -263,7 +266,7 @@ def _create_node(reactor, request, temp_dir, introducer_furl, flog_gatherer, nam
u'log_gatherer.furl',
flog_gatherer.decode("utf-8"),
)
write_config(config_path, config)
write_config(FilePath(config_path), config)
created_d.addCallback(created)
d = Deferred()

0
newsfragments/3511.minor Normal file
View File

133
src/allmydata/node.py
View File

@ -21,9 +21,14 @@ import types
import errno
from base64 import b32decode, b32encode
import attr
# On Python 2 this will be the backported package.
import configparser
from twisted.python.filepath import (
FilePath,
)
from twisted.python import log as twlog
from twisted.application import service
from twisted.python.failure import Failure
@ -190,25 +195,27 @@ def read_config(basedir, portnumfile, generated_files=[], _valid_config=None):
# canonicalize the portnum file
portnumfile = os.path.join(basedir, portnumfile)
# (try to) read the main config file
config_fname = os.path.join(basedir, "tahoe.cfg")
config_path = FilePath(basedir).child("tahoe.cfg")
try:
parser = configutil.get_config(config_fname)
config_str = config_path.getContent()
except EnvironmentError as e:
if e.errno != errno.ENOENT:
raise
# The file is missing, just create empty ConfigParser.
parser = configutil.get_config_from_string(u"")
config_str = u""
else:
config_str = config_str.decode("utf-8-sig")
configutil.validate_config(config_fname, parser, _valid_config)
# make sure we have a private configuration area
fileutil.make_dirs(os.path.join(basedir, "private"), 0o700)
return _Config(parser, portnumfile, basedir, config_fname)
return config_from_string(
basedir,
portnumfile,
config_str,
_valid_config,
config_path,
)
def config_from_string(basedir, portnumfile, config_str, _valid_config=None):
def config_from_string(basedir, portnumfile, config_str, _valid_config=None, fpath=None):
"""
load and validate configuration from in-memory string
"""
@ -221,9 +228,19 @@ def config_from_string(basedir, portnumfile, config_str, _valid_config=None):
# load configuration from in-memory string
parser = configutil.get_config_from_string(config_str)
fname = "<in-memory>"
configutil.validate_config(fname, parser, _valid_config)
return _Config(parser, portnumfile, basedir, fname)
configutil.validate_config(
"<string>" if fpath is None else fpath.path,
parser,
_valid_config,
)
return _Config(
parser,
portnumfile,
basedir,
fpath,
_valid_config,
)
def _error_about_old_config_files(basedir, generated_files):
@ -251,6 +268,7 @@ def _error_about_old_config_files(basedir, generated_files):
raise e
@attr.s
class _Config(object):
"""
Manages configuration of a Tahoe 'node directory'.
@ -259,30 +277,47 @@ class _Config(object):
class; names and funtionality have been kept the same while moving
the code. It probably makes sense for several of these APIs to
have better names.
:ivar ConfigParser config: The actual configuration values.
:ivar str portnum_fname: filename to use for the port-number file (a
relative path inside basedir).
:ivar str _basedir: path to our "node directory", inside which all
configuration is managed.
:ivar (FilePath|NoneType) config_path: The path actually used to create
the configparser (might be ``None`` if using in-memory data).
:ivar ValidConfiguration valid_config_sections: The validator for the
values in this configuration.
"""
config = attr.ib(validator=attr.validators.instance_of(configparser.ConfigParser))
portnum_fname = attr.ib()
_basedir = attr.ib(
converter=lambda basedir: abspath_expanduser_unicode(ensure_text(basedir)),
)
config_path = attr.ib(
validator=attr.validators.optional(
attr.validators.instance_of(FilePath),
),
)
valid_config_sections = attr.ib(
default=configutil.ValidConfiguration.everything(),
validator=attr.validators.instance_of(configutil.ValidConfiguration),
)
def __init__(self, configparser, portnum_fname, basedir, config_fname):
"""
:param configparser: a ConfigParser instance
@property
def nickname(self):
nickname = self.get_config("node", "nickname", u"<unspecified>")
assert isinstance(nickname, str)
return nickname
:param portnum_fname: filename to use for the port-number file
(a relative path inside basedir)
:param basedir: path to our "node directory", inside which all
configuration is managed
:param config_fname: the pathname actually used to create the
configparser (might be 'fake' if using in-memory data)
"""
self.portnum_fname = portnum_fname
self._basedir = abspath_expanduser_unicode(ensure_text(basedir))
self._config_fname = config_fname
self.config = configparser
self.nickname = self.get_config("node", "nickname", u"<unspecified>")
assert isinstance(self.nickname, str)
def validate(self, valid_config_sections):
configutil.validate_config(self._config_fname, self.config, valid_config_sections)
@property
def _config_fname(self):
if self.config_path is None:
return "<string>"
return self.config_path.path
def write_config_file(self, name, value, mode="w"):
"""
@ -327,6 +362,34 @@ class _Config(object):
)
return default
def set_config(self, section, option, value):
"""
Set a config option in a section and re-write the tahoe.cfg file
:param str section: The name of the section in which to set the
option.
:param str option: The name of the option to set.
:param str value: The value of the option.
:raise UnescapedHashError: If the option holds a fURL and there is a
``#`` in the value.
"""
if option.endswith(".furl") and "#" in value:
raise UnescapedHashError(section, option, value)
copied_config = configutil.copy_config(self.config)
configutil.set_config(copied_config, section, option, value)
configutil.validate_config(
self._config_fname,
copied_config,
self.valid_config_sections,
)
if self.config_path is not None:
configutil.write_config(self.config_path, copied_config)
self.config = copied_config
def get_config_from_file(self, name, required=False):
"""Get the (string) contents of a config file, or None if the file
did not exist. If required=True, raise an exception rather than

7
src/allmydata/test/cli/test_create.py
View File

@ -52,13 +52,8 @@ class Config(unittest.TestCase):
create_node.write_node_config(f, opts)
create_node.write_client_config(f, opts)
config = configutil.get_config(fname)
# should succeed, no exceptions
configutil.validate_config(
fname,
config,
client._valid_config(),
)
client.read_config(d, "")
@defer.inlineCallbacks
def test_client(self):

186
src/allmydata/test/test_configutil.py
View File

@ -14,12 +14,89 @@ if PY2:
from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, list, object, range, str, max, min # noqa: F401
import os.path
from configparser import (
ConfigParser,
)
from functools import (
partial,
)
from hypothesis import (
given,
)
from hypothesis.strategies import (
dictionaries,
text,
characters,
)
from twisted.python.filepath import (
FilePath,
)
from twisted.trial import unittest
from allmydata.util import configutil
def arbitrary_config_dicts(
min_sections=0,
max_sections=3,
max_section_name_size=8,
max_items_per_section=3,
max_item_length=8,
max_value_length=8,
):
"""
Build ``dict[str, dict[str, str]]`` instances populated with arbitrary
configurations.
"""
identifier_text = partial(
text,
# Don't allow most control characters or spaces
alphabet=characters(
blacklist_categories=('Cc', 'Cs', 'Zs'),
),
)
return dictionaries(
identifier_text(
min_size=1,
max_size=max_section_name_size,
),
dictionaries(
identifier_text(
min_size=1,
max_size=max_item_length,
),
text(max_size=max_value_length),
max_size=max_items_per_section,
),
min_size=min_sections,
max_size=max_sections,
)
def to_configparser(dictconfig):
"""
Take a ``dict[str, dict[str, str]]`` and turn it into the corresponding
populated ``ConfigParser`` instance.
"""
cp = ConfigParser()
for section, items in dictconfig.items():
cp.add_section(section)
for k, v in items.items():
cp.set(
section,
k,
# ConfigParser has a feature that everyone knows and loves
# where it will use %-style interpolation to substitute
# values from one part of the config into another part of
# the config. Escape all our `%`s to avoid hitting this
# and complicating things.
v.replace("%", "%%"),
)
return cp
class ConfigUtilTests(unittest.TestCase):
def setUp(self):
super(ConfigUtilTests, self).setUp()
@ -55,7 +132,7 @@ enabled = false
# test that set_config can mutate an existing option
configutil.set_config(config, "node", "nickname", "Alice!")
configutil.write_config(tahoe_cfg, config)
configutil.write_config(FilePath(tahoe_cfg), config)
config = configutil.get_config(tahoe_cfg)
self.failUnlessEqual(config.get("node", "nickname"), "Alice!")
@ -63,19 +140,21 @@ enabled = false
# test that set_config can set a new option
descriptor = "Twas brillig, and the slithy toves Did gyre and gimble in the wabe"
configutil.set_config(config, "node", "descriptor", descriptor)
configutil.write_config(tahoe_cfg, config)
configutil.write_config(FilePath(tahoe_cfg), config)
config = configutil.get_config(tahoe_cfg)
self.failUnlessEqual(config.get("node", "descriptor"), descriptor)
def test_config_validation_success(self):
fname = self.create_tahoe_cfg('[node]\nvalid = foo\n')
config = configutil.get_config(fname)
"""
``configutil.validate_config`` returns ``None`` when the configuration it
is given has nothing more than the static sections and items defined
by the validator.
"""
# should succeed, no exceptions
configutil.validate_config(
fname,
config,
"<test_config_validation_success>",
to_configparser({"node": {"valid": "foo"}}),
self.static_valid_config,
)
@ -85,24 +164,20 @@ enabled = false
validation but are matched by the dynamic validation is considered
valid.
"""
fname = self.create_tahoe_cfg('[node]\nvalid = foo\n')
config = configutil.get_config(fname)
# should succeed, no exceptions
configutil.validate_config(
fname,
config,
"<test_config_dynamic_validation_success>",
to_configparser({"node": {"valid": "foo"}}),
self.dynamic_valid_config,
)
def test_config_validation_invalid_item(self):
fname = self.create_tahoe_cfg('[node]\nvalid = foo\ninvalid = foo\n')
config = configutil.get_config(fname)
config = to_configparser({"node": {"valid": "foo", "invalid": "foo"}})
e = self.assertRaises(
configutil.UnknownConfigError,
configutil.validate_config,
fname, config,
"<test_config_validation_invalid_item>",
config,
self.static_valid_config,
)
self.assertIn("section [node] contains unknown option 'invalid'", str(e))
@ -112,13 +187,12 @@ enabled = false
A configuration with a section that is matched by neither the static nor
dynamic validators is rejected.
"""
fname = self.create_tahoe_cfg('[node]\nvalid = foo\n[invalid]\n')
config = configutil.get_config(fname)
config = to_configparser({"node": {"valid": "foo"}, "invalid": {}})
e = self.assertRaises(
configutil.UnknownConfigError,
configutil.validate_config,
fname, config,
"<test_config_validation_invalid_section>",
config,
self.static_valid_config,
)
self.assertIn("contains unknown section [invalid]", str(e))
@ -128,13 +202,12 @@ enabled = false
A configuration with a section that is matched by neither the static nor
dynamic validators is rejected.
"""
fname = self.create_tahoe_cfg('[node]\nvalid = foo\n[invalid]\n')
config = configutil.get_config(fname)
config = to_configparser({"node": {"valid": "foo"}, "invalid": {}})
e = self.assertRaises(
configutil.UnknownConfigError,
configutil.validate_config,
fname, config,
"<test_config_dynamic_validation_invalid_section>",
config,
self.dynamic_valid_config,
)
self.assertIn("contains unknown section [invalid]", str(e))
@ -144,13 +217,12 @@ enabled = false
A configuration with a section, item pair that is matched by neither the
static nor dynamic validators is rejected.
"""
fname = self.create_tahoe_cfg('[node]\nvalid = foo\ninvalid = foo\n')
config = configutil.get_config(fname)
config = to_configparser({"node": {"valid": "foo", "invalid": "foo"}})
e = self.assertRaises(
configutil.UnknownConfigError,
configutil.validate_config,
fname, config,
"<test_config_dynamic_validation_invalid_item>",
config,
self.dynamic_valid_config,
)
self.assertIn("section [node] contains unknown option 'invalid'", str(e))
@ -163,3 +235,61 @@ enabled = false
config = configutil.get_config(fname)
self.assertEqual(config.get("node", "a"), "foo")
self.assertEqual(config.get("node", "b"), "bar")
@given(arbitrary_config_dicts())
def test_everything_valid(self, cfgdict):
"""
``validate_config`` returns ``None`` when the validator is
``ValidConfiguration.everything()``.
"""
cfg = to_configparser(cfgdict)
self.assertIs(
configutil.validate_config(
"<test_everything_valid>",
cfg,
configutil.ValidConfiguration.everything(),
),
None,
)
@given(arbitrary_config_dicts(min_sections=1))
def test_nothing_valid(self, cfgdict):
"""
``validate_config`` raises ``UnknownConfigError`` when the validator is
``ValidConfiguration.nothing()`` for all non-empty configurations.
"""
cfg = to_configparser(cfgdict)
with self.assertRaises(configutil.UnknownConfigError):
configutil.validate_config(
"<test_everything_valid>",
cfg,
configutil.ValidConfiguration.nothing(),
)
def test_nothing_empty_valid(self):
"""
``validate_config`` returns ``None`` when the validator is
``ValidConfiguration.nothing()`` if the configuration is empty.
"""
cfg = ConfigParser()
self.assertIs(
configutil.validate_config(
"<test_everything_valid>",
cfg,
configutil.ValidConfiguration.nothing(),
),
None,
)
@given(arbitrary_config_dicts())
def test_copy_config(self, cfgdict):
"""
``copy_config`` creates a new ``ConfigParser`` object containing the same
values as its input.
"""
cfg = to_configparser(cfgdict)
copied = configutil.copy_config(cfg)
# Should be equal
self.assertEqual(cfg, copied)
# But not because they're the same object.
self.assertIsNot(cfg, copied)

81
src/allmydata/test/test_node.py
View File

@ -29,6 +29,9 @@ from hypothesis.strategies import (
from unittest import skipIf
from twisted.python.filepath import (
FilePath,
)
from twisted.trial import unittest
from twisted.internet import defer
@ -52,7 +55,11 @@ from allmydata import client
from allmydata.util import fileutil, iputil
from allmydata.util.namespace import Namespace
from allmydata.util.configutil import UnknownConfigError
from allmydata.util.configutil import (
ValidConfiguration,
UnknownConfigError,
)
from allmydata.util.i2p_provider import create as create_i2p_provider
from allmydata.util.tor_provider import create as create_tor_provider
import allmydata.test.common_util as testutil
@ -431,6 +438,78 @@ class TestCase(testutil.SignalMixin, unittest.TestCase):
yield client.create_client(basedir)
self.failUnless(ns.called)
def test_set_config_unescaped_furl_hash(self):
"""
``_Config.set_config`` raises ``UnescapedHashError`` if the item being set
is a furl and the value includes ``"#"`` and does not set the value.
"""
basedir = self.mktemp()
new_config = config_from_string(basedir, "", "")
with self.assertRaises(UnescapedHashError):
new_config.set_config("foo", "bar.furl", "value#1")
with self.assertRaises(MissingConfigEntry):
new_config.get_config("foo", "bar.furl")
def test_set_config_new_section(self):
"""
``_Config.set_config`` can be called with the name of a section that does
not already exist to create that section and set an item in it.
"""
basedir = self.mktemp()
new_config = config_from_string(basedir, "", "", ValidConfiguration.everything())
new_config.set_config("foo", "bar", "value1")
self.assertEqual(
new_config.get_config("foo", "bar"),
"value1"
)
def test_set_config_replace(self):
"""
``_Config.set_config`` can be called with a section and item that already
exists to change an existing value to a new one.
"""
basedir = self.mktemp()
new_config = config_from_string(basedir, "", "", ValidConfiguration.everything())
new_config.set_config("foo", "bar", "value1")
new_config.set_config("foo", "bar", "value2")
self.assertEqual(
new_config.get_config("foo", "bar"),
"value2"
)
def test_set_config_write(self):
"""
``_Config.set_config`` persists the configuration change so it can be
re-loaded later.
"""
# Let our nonsense config through
valid_config = ValidConfiguration.everything()
basedir = FilePath(self.mktemp())
basedir.makedirs()
cfg = basedir.child(b"tahoe.cfg")
cfg.setContent(b"")
new_config = read_config(basedir.path, "", [], valid_config)
new_config.set_config("foo", "bar", "value1")
loaded_config = read_config(basedir.path, "", [], valid_config)
self.assertEqual(
loaded_config.get_config("foo", "bar"),
"value1",
)
def test_set_config_rejects_invalid_config(self):
"""
``_Config.set_config`` raises ``UnknownConfigError`` if the section or
item is not recognized by the validation object and does not set the
value.
"""
# Make everything invalid.
valid_config = ValidConfiguration.nothing()
new_config = config_from_string(self.mktemp(), "", "", valid_config)
with self.assertRaises(UnknownConfigError):
new_config.set_config("foo", "bar", "baz")
with self.assertRaises(MissingConfigEntry):
new_config.get_config("foo", "bar")
class TestMissingPorts(unittest.TestCase):
"""

68
src/allmydata/util/configutil.py
View File

@ -20,6 +20,10 @@ from configparser import ConfigParser
import attr
from twisted.python.runtime import (
platform,
)
class UnknownConfigError(Exception):
"""
@ -59,8 +63,25 @@ def set_config(config, section, option, value):
assert config.get(section, option) == value
def write_config(tahoe_cfg, config):
with open(tahoe_cfg, "w") as f:
config.write(f)
"""
Write a configuration to a file.
:param FilePath tahoe_cfg: The path to which to write the config.
:param ConfigParser config: The configuration to write.
:return: ``None``
"""
tmp = tahoe_cfg.temporarySibling()
# FilePath.open can only open files in binary mode which does not work
# with ConfigParser.write.
with open(tmp.path, "wt") as fp:
config.write(fp)
# Windows doesn't have atomic overwrite semantics for moveTo. Thus we end
# up slightly less than atomic.
if platform.isWindows():
tahoe_cfg.remove()
tmp.moveTo(tahoe_cfg)
def validate_config(fname, cfg, valid_config):
"""
@ -102,10 +123,34 @@ class ValidConfiguration(object):
an item name as bytes and returns True if that section, item pair is
valid, False otherwise.
"""
_static_valid_sections = attr.ib()
_static_valid_sections = attr.ib(
validator=attr.validators.instance_of(dict)
)
_is_valid_section = attr.ib(default=lambda section_name: False)
_is_valid_item = attr.ib(default=lambda section_name, item_name: False)
@classmethod
def everything(cls):
"""
Create a validator which considers everything valid.
"""
return cls(
{},
lambda section_name: True,
lambda section_name, item_name: True,
)
@classmethod
def nothing(cls):
"""
Create a validator which considers nothing valid.
"""
return cls(
{},
lambda section_name: False,
lambda section_name, item_name: False,
)
def is_valid_section(self, section_name):
"""
:return: True if the given section name is valid, False otherwise.
@ -136,6 +181,23 @@ class ValidConfiguration(object):
)
def copy_config(old):
"""
Return a brand new ``ConfigParser`` containing the same values as
the given object.
:param ConfigParser old: The configuration to copy.
:return ConfigParser: The new object containing the same configuration.
"""
new = ConfigParser()
for section_name in old.sections():
new.add_section(section_name)
for k, v in old.items(section_name):
new.set(section_name, k, v.replace("%", "%%"))
return new
def _either(f, g):
"""
:return: A function which returns True if either f or g returns True.