From 96b54e8f621475a73ac2f0699fc5ca386b36a449 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 16 Dec 2020 20:50:08 -0500 Subject: [PATCH 001/269] news fragment --- newsfragments/3528.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3528.minor diff --git a/newsfragments/3528.minor b/newsfragments/3528.minor new file mode 100644 index 000000000..e69de29bb From d5bff458b6ffcc220ea5b2624f9b7c151ec99707 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 16 Dec 2020 20:51:01 -0500 Subject: [PATCH 002/269] Parameterize argv to allmydata.scripts.runner.run --- src/allmydata/scripts/runner.py | 20 ++++++++++---------- src/allmydata/test/cli/test_cli.py | 9 ++++++--- src/allmydata/test/common_util.py | 2 +- 3 files changed, 17 insertions(+), 14 deletions(-) diff --git a/src/allmydata/scripts/runner.py b/src/allmydata/scripts/runner.py index 1f993fda1..c50b6ae99 100644 --- a/src/allmydata/scripts/runner.py +++ b/src/allmydata/scripts/runner.py @@ -109,7 +109,7 @@ def parse_options(argv, config=None): def parse_or_exit_with_explanation(argv, stdout=sys.stdout): config = Options() try: - parse_options(argv, config=config) + parse_options(argv[1:], config=config) except usage.error as e: c = config while hasattr(c, 'subOptions'): @@ -119,7 +119,7 @@ def parse_or_exit_with_explanation(argv, stdout=sys.stdout): msg = e.args[0].decode(get_io_encoding()) except Exception: msg = repr(e) - print("%s: %s\n" % (sys.argv[0], quote_output(msg, quotemarks=False)), file=stdout) + print("%s: %s\n" % (argv[0], quote_output(msg, quotemarks=False)), file=stdout) sys.exit(1) return config @@ -171,7 +171,7 @@ def _maybe_enable_eliot_logging(options, reactor): # Pass on the options so we can dispatch the subcommand. return options -def run(): +def run(argv=sys.argv): # TODO(3035): Remove tox-check when error becomes a warning if 'TOX_ENV_NAME' not in os.environ: assert sys.version_info < (3,), u"Tahoe-LAFS does not run under Python 3. Please use Python 2.7.x." @@ -180,19 +180,19 @@ def run(): from allmydata.windows.fixups import initialize initialize() # doesn't return: calls sys.exit(rc) - task.react(_run_with_reactor) + task.react(_run_with_reactor, argv) -def _setup_coverage(reactor): +def _setup_coverage(reactor, argv): """ Arrange for coverage to be collected if the 'coverage' package is installed """ # can we put this _setup_coverage call after we hit # argument-parsing? - if '--coverage' not in sys.argv: + if '--coverage' not in argv: return - sys.argv.remove('--coverage') + argv.remove('--coverage') try: import coverage @@ -223,11 +223,11 @@ def _setup_coverage(reactor): reactor.addSystemEventTrigger('after', 'shutdown', write_coverage_data) -def _run_with_reactor(reactor): +def _run_with_reactor(reactor, argv): - _setup_coverage(reactor) + _setup_coverage(reactor, argv) - d = defer.maybeDeferred(parse_or_exit_with_explanation, sys.argv[1:]) + d = defer.maybeDeferred(parse_or_exit_with_explanation, argv) d.addCallback(_maybe_enable_eliot_logging, reactor) d.addCallback(dispatch) def _show_exception(f): diff --git a/src/allmydata/test/cli/test_cli.py b/src/allmydata/test/cli/test_cli.py index 2b1bc1c86..2a19e65f7 100644 --- a/src/allmydata/test/cli/test_cli.py +++ b/src/allmydata/test/cli/test_cli.py @@ -524,7 +524,7 @@ class CLI(CLITestMixin, unittest.TestCase): ns.sys_exit_called = True self.failUnlessEqual(exitcode, 1) - def fake_react(f): + def fake_react(f, *args): reactor = Mock() d = f(reactor) # normally this Deferred would be errbacked with SystemExit, but @@ -534,12 +534,15 @@ class CLI(CLITestMixin, unittest.TestCase): patcher = MonkeyPatcher((runner, 'parse_or_exit_with_explanation', call_parse_or_exit), - (sys, 'argv', ["tahoe"]), (sys, 'exit', call_sys_exit), (sys, 'stderr', stderr), (task, 'react', fake_react), ) - patcher.runWithPatches(runner.run) + patcher.runWithPatches( + lambda: runner.run( + ["tahoe"], + ), + ) self.failUnless(ns.parse_called) self.failUnless(ns.sys_exit_called) diff --git a/src/allmydata/test/common_util.py b/src/allmydata/test/common_util.py index 341d383c1..6a221f509 100644 --- a/src/allmydata/test/common_util.py +++ b/src/allmydata/test/common_util.py @@ -81,7 +81,7 @@ def run_cli_bytes(verb, *args, **kwargs): args=args, nodeargs=nodeargs, ) - argv = nodeargs + [verb] + list(args) + argv = ["tahoe"] + nodeargs + [verb] + list(args) stdin = kwargs.get("stdin", "") if encoding is None: # The original behavior, the Python 2 behavior, is to accept either From 1f229ce9f6571bdd972cd75ff361a3a65c209058 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 16 Dec 2020 20:51:11 -0500 Subject: [PATCH 003/269] All you have to do to drop it is not save it in the first place Also it would have been dropped as soon as this function returned, anyway. --- src/allmydata/test/cli/test_cli.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/allmydata/test/cli/test_cli.py b/src/allmydata/test/cli/test_cli.py index 2a19e65f7..b8eab763e 100644 --- a/src/allmydata/test/cli/test_cli.py +++ b/src/allmydata/test/cli/test_cli.py @@ -526,11 +526,10 @@ class CLI(CLITestMixin, unittest.TestCase): def fake_react(f, *args): reactor = Mock() - d = f(reactor) # normally this Deferred would be errbacked with SystemExit, but # since we mocked out sys.exit, it will be fired with None. So # it's safe to drop it on the floor. - del d + f(reactor, *args) patcher = MonkeyPatcher((runner, 'parse_or_exit_with_explanation', call_parse_or_exit), From a4b0b4a01ab964e286e593d373480674ab629beb Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 16 Dec 2020 20:55:00 -0500 Subject: [PATCH 004/269] Parameterize stderr to allmydata.scripts.runner.run --- src/allmydata/scripts/runner.py | 8 ++++---- src/allmydata/test/cli/test_cli.py | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/allmydata/scripts/runner.py b/src/allmydata/scripts/runner.py index c50b6ae99..9843edb6e 100644 --- a/src/allmydata/scripts/runner.py +++ b/src/allmydata/scripts/runner.py @@ -171,7 +171,7 @@ def _maybe_enable_eliot_logging(options, reactor): # Pass on the options so we can dispatch the subcommand. return options -def run(argv=sys.argv): +def run(argv=sys.argv, stderr=sys.stderr): # TODO(3035): Remove tox-check when error becomes a warning if 'TOX_ENV_NAME' not in os.environ: assert sys.version_info < (3,), u"Tahoe-LAFS does not run under Python 3. Please use Python 2.7.x." @@ -180,7 +180,7 @@ def run(argv=sys.argv): from allmydata.windows.fixups import initialize initialize() # doesn't return: calls sys.exit(rc) - task.react(_run_with_reactor, argv) + task.react(_run_with_reactor, argv, stderr) def _setup_coverage(reactor, argv): @@ -223,7 +223,7 @@ def _setup_coverage(reactor, argv): reactor.addSystemEventTrigger('after', 'shutdown', write_coverage_data) -def _run_with_reactor(reactor, argv): +def _run_with_reactor(reactor, argv, stderr): _setup_coverage(reactor, argv) @@ -237,7 +237,7 @@ def _run_with_reactor(reactor, argv): # weren't using react(). if f.check(SystemExit): return f # dispatch function handled it - f.printTraceback(file=sys.stderr) + f.printTraceback(file=stderr) sys.exit(1) d.addErrback(_show_exception) return d diff --git a/src/allmydata/test/cli/test_cli.py b/src/allmydata/test/cli/test_cli.py index b8eab763e..9c3958704 100644 --- a/src/allmydata/test/cli/test_cli.py +++ b/src/allmydata/test/cli/test_cli.py @@ -534,12 +534,12 @@ class CLI(CLITestMixin, unittest.TestCase): patcher = MonkeyPatcher((runner, 'parse_or_exit_with_explanation', call_parse_or_exit), (sys, 'exit', call_sys_exit), - (sys, 'stderr', stderr), (task, 'react', fake_react), ) patcher.runWithPatches( lambda: runner.run( - ["tahoe"], + argv=["tahoe"], + stderr=stderr, ), ) From 2746eb9ae1919084218ae66fb439d20e106996ec Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 16 Dec 2020 20:58:27 -0500 Subject: [PATCH 005/269] Fix the broken fake_react by not using the argv feature --- src/allmydata/scripts/runner.py | 2 +- src/allmydata/test/cli/test_cli.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/allmydata/scripts/runner.py b/src/allmydata/scripts/runner.py index 9843edb6e..8b730d9cd 100644 --- a/src/allmydata/scripts/runner.py +++ b/src/allmydata/scripts/runner.py @@ -180,7 +180,7 @@ def run(argv=sys.argv, stderr=sys.stderr): from allmydata.windows.fixups import initialize initialize() # doesn't return: calls sys.exit(rc) - task.react(_run_with_reactor, argv, stderr) + task.react(lambda reactor: _run_with_reactor(reactor, argv, stderr)) def _setup_coverage(reactor, argv): diff --git a/src/allmydata/test/cli/test_cli.py b/src/allmydata/test/cli/test_cli.py index 9c3958704..d1bcfc128 100644 --- a/src/allmydata/test/cli/test_cli.py +++ b/src/allmydata/test/cli/test_cli.py @@ -524,12 +524,12 @@ class CLI(CLITestMixin, unittest.TestCase): ns.sys_exit_called = True self.failUnlessEqual(exitcode, 1) - def fake_react(f, *args): + def fake_react(f): reactor = Mock() # normally this Deferred would be errbacked with SystemExit, but # since we mocked out sys.exit, it will be fired with None. So # it's safe to drop it on the floor. - f(reactor, *args) + f(reactor) patcher = MonkeyPatcher((runner, 'parse_or_exit_with_explanation', call_parse_or_exit), From a04a915628f6a532eebcccdc0d8208e10a964965 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 16 Dec 2020 21:15:24 -0500 Subject: [PATCH 006/269] Parameterize the Options class so we can synthesize an unhandled exception --- src/allmydata/scripts/runner.py | 27 ++++++++++++++++++++------- src/allmydata/test/cli/test_cli.py | 17 +++++++---------- src/allmydata/test/common_util.py | 21 +++++++++++++++++---- 3 files changed, 44 insertions(+), 21 deletions(-) diff --git a/src/allmydata/scripts/runner.py b/src/allmydata/scripts/runner.py index 8b730d9cd..bc887e11a 100644 --- a/src/allmydata/scripts/runner.py +++ b/src/allmydata/scripts/runner.py @@ -106,8 +106,7 @@ def parse_options(argv, config=None): config.parseOptions(argv) # may raise usage.error return config -def parse_or_exit_with_explanation(argv, stdout=sys.stdout): - config = Options() +def parse_or_exit_with_explanation_with_config(config, argv, stdout, stderr): try: parse_options(argv[1:], config=config) except usage.error as e: @@ -171,7 +170,7 @@ def _maybe_enable_eliot_logging(options, reactor): # Pass on the options so we can dispatch the subcommand. return options -def run(argv=sys.argv, stderr=sys.stderr): +def run(configFactory=Options, argv=sys.argv, stdout=sys.stdout, stderr=sys.stderr): # TODO(3035): Remove tox-check when error becomes a warning if 'TOX_ENV_NAME' not in os.environ: assert sys.version_info < (3,), u"Tahoe-LAFS does not run under Python 3. Please use Python 2.7.x." @@ -180,7 +179,15 @@ def run(argv=sys.argv, stderr=sys.stderr): from allmydata.windows.fixups import initialize initialize() # doesn't return: calls sys.exit(rc) - task.react(lambda reactor: _run_with_reactor(reactor, argv, stderr)) + task.react( + lambda reactor: _run_with_reactor( + reactor, + configFactory(), + argv, + stdout, + stderr, + ), + ) def _setup_coverage(reactor, argv): @@ -223,13 +230,19 @@ def _setup_coverage(reactor, argv): reactor.addSystemEventTrigger('after', 'shutdown', write_coverage_data) -def _run_with_reactor(reactor, argv, stderr): +def _run_with_reactor(reactor, config, argv, stdout, stderr): _setup_coverage(reactor, argv) - d = defer.maybeDeferred(parse_or_exit_with_explanation, argv) + d = defer.maybeDeferred( + parse_or_exit_with_explanation_with_config, + config, + argv, + stdout, + stderr, + ) d.addCallback(_maybe_enable_eliot_logging, reactor) - d.addCallback(dispatch) + d.addCallback(dispatch, stdout=stdout, stderr=stderr) def _show_exception(f): # when task.react() notices a non-SystemExit exception, it does # log.err() with the failure and then exits with rc=1. We want this diff --git a/src/allmydata/test/cli/test_cli.py b/src/allmydata/test/cli/test_cli.py index d1bcfc128..f0e6fd2e4 100644 --- a/src/allmydata/test/cli/test_cli.py +++ b/src/allmydata/test/cli/test_cli.py @@ -510,14 +510,13 @@ class CLI(CLITestMixin, unittest.TestCase): def test_exception_catcher(self): self.basedir = "cli/exception_catcher" - stderr = StringIO() exc = Exception("canary") - ns = Namespace() + class BrokenOptions(object): + def parseOptions(self, argv): + raise exc - ns.parse_called = False - def call_parse_or_exit(args): - ns.parse_called = True - raise exc + stderr = StringIO() + ns = Namespace() ns.sys_exit_called = False def call_sys_exit(exitcode): @@ -531,19 +530,17 @@ class CLI(CLITestMixin, unittest.TestCase): # it's safe to drop it on the floor. f(reactor) - patcher = MonkeyPatcher((runner, 'parse_or_exit_with_explanation', - call_parse_or_exit), - (sys, 'exit', call_sys_exit), + patcher = MonkeyPatcher((sys, 'exit', call_sys_exit), (task, 'react', fake_react), ) patcher.runWithPatches( lambda: runner.run( + configFactory=BrokenOptions, argv=["tahoe"], stderr=stderr, ), ) - self.failUnless(ns.parse_called) self.failUnless(ns.sys_exit_called) self.failUnlessIn(str(exc), stderr.getvalue()) diff --git a/src/allmydata/test/common_util.py b/src/allmydata/test/common_util.py index 6a221f509..8885e067e 100644 --- a/src/allmydata/test/common_util.py +++ b/src/allmydata/test/common_util.py @@ -3,6 +3,9 @@ from __future__ import print_function import os import time import signal +from functools import ( + partial, +) from random import randrange from six.moves import StringIO from io import ( @@ -100,10 +103,20 @@ def run_cli_bytes(verb, *args, **kwargs): stdout = TextIOWrapper(BytesIO(), encoding) stderr = TextIOWrapper(BytesIO(), encoding) d = defer.succeed(argv) - d.addCallback(runner.parse_or_exit_with_explanation, stdout=stdout) - d.addCallback(runner.dispatch, - stdin=StringIO(stdin), - stdout=stdout, stderr=stderr) + d.addCallback( + partial( + runner.parse_or_exit_with_explanation_with_config, + runner.Options(), + ), + stdout=stdout, + stderr=stderr, + ) + d.addCallback( + runner.dispatch, + stdin=StringIO(stdin), + stdout=stdout, + stderr=stderr, + ) def _done(rc): return 0, _getvalue(stdout), _getvalue(stderr) def _err(f): From faf8da82dd2c6bca84a0c4541ed86b53183c1d8b Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 16 Dec 2020 21:20:16 -0500 Subject: [PATCH 007/269] Get rid of the sys.exit monkey-patch It's just an exception. Let it get logged and then check after. --- src/allmydata/test/cli/test_cli.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/src/allmydata/test/cli/test_cli.py b/src/allmydata/test/cli/test_cli.py index f0e6fd2e4..520461232 100644 --- a/src/allmydata/test/cli/test_cli.py +++ b/src/allmydata/test/cli/test_cli.py @@ -516,12 +516,6 @@ class CLI(CLITestMixin, unittest.TestCase): raise exc stderr = StringIO() - ns = Namespace() - - ns.sys_exit_called = False - def call_sys_exit(exitcode): - ns.sys_exit_called = True - self.failUnlessEqual(exitcode, 1) def fake_react(f): reactor = Mock() @@ -530,8 +524,7 @@ class CLI(CLITestMixin, unittest.TestCase): # it's safe to drop it on the floor. f(reactor) - patcher = MonkeyPatcher((sys, 'exit', call_sys_exit), - (task, 'react', fake_react), + patcher = MonkeyPatcher((task, 'react', fake_react), ) patcher.runWithPatches( lambda: runner.run( @@ -541,8 +534,9 @@ class CLI(CLITestMixin, unittest.TestCase): ), ) - self.failUnless(ns.sys_exit_called) self.failUnlessIn(str(exc), stderr.getvalue()) + [exit_exc] = self.flushLoggedErrors(SystemExit) + self.assertEqual(1, exit_exc.value.code) class Help(unittest.TestCase): From 240d5d11643f8af352e3d7a41ec0464695027eaf Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 16 Dec 2020 21:25:50 -0500 Subject: [PATCH 008/269] Remove react monkey patching by supplying an alternate reactor Let react run and do its thing. This gives us an even nicer way to check the exit code. --- src/allmydata/test/cli/test_cli.py | 40 ++++++++++++++++-------------- 1 file changed, 22 insertions(+), 18 deletions(-) diff --git a/src/allmydata/test/cli/test_cli.py b/src/allmydata/test/cli/test_cli.py index 520461232..7720d555c 100644 --- a/src/allmydata/test/cli/test_cli.py +++ b/src/allmydata/test/cli/test_cli.py @@ -8,7 +8,12 @@ from twisted.trial import unittest from twisted.python.monkey import MonkeyPatcher from twisted.internet import task from twisted.python.filepath import FilePath - +from twisted.internet.testing import ( + MemoryReactor, +) +from twisted.internet.test.modulehelpers import ( + AlternateReactor, +) import allmydata from allmydata.crypto import ed25519 from allmydata.util import fileutil, hashutil, base32 @@ -508,6 +513,10 @@ class CLI(CLITestMixin, unittest.TestCase): self.failUnlessIn(normalize(file), filenames) def test_exception_catcher(self): + """ + An exception that is otherwise unhandled during argument dispatch is + written to stderr and causes the process to exit with code 1. + """ self.basedir = "cli/exception_catcher" exc = Exception("canary") @@ -517,26 +526,21 @@ class CLI(CLITestMixin, unittest.TestCase): stderr = StringIO() - def fake_react(f): - reactor = Mock() - # normally this Deferred would be errbacked with SystemExit, but - # since we mocked out sys.exit, it will be fired with None. So - # it's safe to drop it on the floor. - f(reactor) + reactor = MemoryReactor() - patcher = MonkeyPatcher((task, 'react', fake_react), - ) - patcher.runWithPatches( - lambda: runner.run( - configFactory=BrokenOptions, - argv=["tahoe"], - stderr=stderr, - ), - ) + with AlternateReactor(reactor): + with self.assertRaises(SystemExit) as ctx: + runner.run( + configFactory=BrokenOptions, + argv=["tahoe"], + stderr=stderr, + ) + + self.assertTrue(reactor.hasRun) + self.assertFalse(reactor.running) self.failUnlessIn(str(exc), stderr.getvalue()) - [exit_exc] = self.flushLoggedErrors(SystemExit) - self.assertEqual(1, exit_exc.value.code) + self.assertEqual(1, ctx.exception.code) class Help(unittest.TestCase): From bb495b6dc5c339fff553466f4b58d4644577f5eb Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 16 Dec 2020 21:26:59 -0500 Subject: [PATCH 009/269] unused imports --- src/allmydata/test/cli/test_cli.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/allmydata/test/cli/test_cli.py b/src/allmydata/test/cli/test_cli.py index 7720d555c..8d9c60a6c 100644 --- a/src/allmydata/test/cli/test_cli.py +++ b/src/allmydata/test/cli/test_cli.py @@ -1,12 +1,10 @@ import os.path from six.moves import cStringIO as StringIO -import urllib, sys +import urllib import re -from mock import patch, Mock +from mock import patch from twisted.trial import unittest -from twisted.python.monkey import MonkeyPatcher -from twisted.internet import task from twisted.python.filepath import FilePath from twisted.internet.testing import ( MemoryReactor, @@ -17,7 +15,6 @@ from twisted.internet.test.modulehelpers import ( import allmydata from allmydata.crypto import ed25519 from allmydata.util import fileutil, hashutil, base32 -from allmydata.util.namespace import Namespace from allmydata import uri from allmydata.immutable import upload from allmydata.dirnode import normalize From a363c8de67eaa3aefd85370be35afd7f2c8a7857 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 17 Dec 2020 09:11:29 -0500 Subject: [PATCH 010/269] Fix test_non_numeric_pid (and put it in a better place too) --- src/allmydata/scripts/tahoe_run.py | 9 +++---- src/allmydata/test/cli/test_cli.py | 29 ---------------------- src/allmydata/test/cli/test_run.py | 39 ++++++++++++++++++++++++++++++ 3 files changed, 42 insertions(+), 35 deletions(-) diff --git a/src/allmydata/scripts/tahoe_run.py b/src/allmydata/scripts/tahoe_run.py index bc4ba27d1..071dfac60 100644 --- a/src/allmydata/scripts/tahoe_run.py +++ b/src/allmydata/scripts/tahoe_run.py @@ -182,7 +182,7 @@ class DaemonizeTahoeNodePlugin(object): return DaemonizeTheRealService(self.nodetype, self.basedir, so) -def run(config): +def run(config, runApp=twistd.runApp): """ Runs a Tahoe-LAFS node in the foreground. @@ -202,10 +202,7 @@ def run(config): if not nodetype: print("%s is not a recognizable node directory" % quoted_basedir, file=err) return 1 - # Now prepare to turn into a twistd process. This os.chdir is the point - # of no return. - os.chdir(basedir) - twistd_args = ["--nodaemon"] + twistd_args = ["--nodaemon", "--rundir", basedir] twistd_args.extend(config.twistd_args) twistd_args.append("DaemonizeTahoeNode") # point at our DaemonizeTahoeNodePlugin @@ -229,5 +226,5 @@ def run(config): # We always pass --nodaemon so twistd.runApp does not daemonize. print("running node in %s" % (quoted_basedir,), file=out) - twistd.runApp(twistd_config) + runApp(twistd_config) return 0 diff --git a/src/allmydata/test/cli/test_cli.py b/src/allmydata/test/cli/test_cli.py index 8d9c60a6c..a851790e6 100644 --- a/src/allmydata/test/cli/test_cli.py +++ b/src/allmydata/test/cli/test_cli.py @@ -2,10 +2,8 @@ import os.path from six.moves import cStringIO as StringIO import urllib import re -from mock import patch from twisted.trial import unittest -from twisted.python.filepath import FilePath from twisted.internet.testing import ( MemoryReactor, ) @@ -1308,30 +1306,3 @@ class Options(ReallyEqualMixin, unittest.TestCase): ["--node-directory=there", "run", some_twistd_option]) self.failUnlessRaises(usage.UsageError, self.parse, ["run", "--basedir=here", some_twistd_option]) - - -class Run(unittest.TestCase): - - @patch('allmydata.scripts.tahoe_run.os.chdir') - @patch('allmydata.scripts.tahoe_run.twistd') - def test_non_numeric_pid(self, mock_twistd, chdir): - """ - If the pidfile exists but does not contain a numeric value, a complaint to - this effect is written to stderr. - """ - basedir = FilePath(self.mktemp().decode("ascii")) - basedir.makedirs() - basedir.child(u"twistd.pid").setContent(b"foo") - basedir.child(u"tahoe-client.tac").setContent(b"") - - config = tahoe_run.RunOptions() - config.stdout = StringIO() - config.stderr = StringIO() - config['basedir'] = basedir.path - config.twistd_args = [] - - result_code = tahoe_run.run(config) - self.assertIn("invalid PID file", config.stderr.getvalue()) - self.assertTrue(len(mock_twistd.mock_calls), 1) - self.assertEqual(mock_twistd.mock_calls[0][0], 'runApp') - self.assertEqual(0, result_code) diff --git a/src/allmydata/test/cli/test_run.py b/src/allmydata/test/cli/test_run.py index d27791f34..e4bcdfbcd 100644 --- a/src/allmydata/test/cli/test_run.py +++ b/src/allmydata/test/cli/test_run.py @@ -9,6 +9,7 @@ from six.moves import ( from testtools.matchers import ( Contains, Equals, + HasLength, ) from twisted.python.filepath import ( @@ -23,6 +24,8 @@ from twisted.internet.test.modulehelpers import ( from ...scripts.tahoe_run import ( DaemonizeTheRealService, + RunOptions, + run, ) from ...scripts.runner import ( @@ -125,3 +128,39 @@ class DaemonizeTheRealServiceTests(SyncTestCase): """, "Privacy requested", ) + + +class RunTests(SyncTestCase): + """ + Tests for ``run``. + """ + def test_non_numeric_pid(self): + """ + If the pidfile exists but does not contain a numeric value, a complaint to + this effect is written to stderr. + """ + basedir = FilePath(self.mktemp().decode("ascii")) + basedir.makedirs() + basedir.child(u"twistd.pid").setContent(b"foo") + basedir.child(u"tahoe-client.tac").setContent(b"") + + config = RunOptions() + config.stdout = StringIO() + config.stderr = StringIO() + config['basedir'] = basedir.path + config.twistd_args = [] + + runs = [] + result_code = run(config, runApp=runs.append) + self.assertThat( + config.stderr.getvalue(), + Contains("found invalid PID file in"), + ) + self.assertThat( + runs, + HasLength(1), + ) + self.assertThat( + result_code, + Equals(0), + ) From f88061e31c5f643441b558ed8c72329cd9f54f0b Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 17 Dec 2020 09:16:05 -0500 Subject: [PATCH 011/269] docstring --- src/allmydata/scripts/runner.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/src/allmydata/scripts/runner.py b/src/allmydata/scripts/runner.py index bc887e11a..4d8601828 100644 --- a/src/allmydata/scripts/runner.py +++ b/src/allmydata/scripts/runner.py @@ -107,6 +107,25 @@ def parse_options(argv, config=None): return config def parse_or_exit_with_explanation_with_config(config, argv, stdout, stderr): + """ + Parse Tahoe-LAFS CLI arguments and return a configuration object if they + are valid. + + If they are invalid, write an explanation to ``stdout`` and exit. + + :param twisted.python.usage.Options config: An instance of the + argument-parsing class to use. + + :param [str] argv: The argument list to parse, including the name of the + program being run as ``argv[0]``. + + :param stdout: The file-like object to use as stdout. + :param stderr: The file-like object to use as stderr. + + :raise SystemExit: If there is an argument-parsing problem. + + :return: ``config``, after using it to parse the argument list. + """ try: parse_options(argv[1:], config=config) except usage.error as e: From 70305131f15c195a9522ee7692dbd30609b28de9 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 17 Dec 2020 09:22:43 -0500 Subject: [PATCH 012/269] docstrings --- src/allmydata/scripts/runner.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/src/allmydata/scripts/runner.py b/src/allmydata/scripts/runner.py index 4d8601828..5b8e2bd4a 100644 --- a/src/allmydata/scripts/runner.py +++ b/src/allmydata/scripts/runner.py @@ -190,6 +190,19 @@ def _maybe_enable_eliot_logging(options, reactor): return options def run(configFactory=Options, argv=sys.argv, stdout=sys.stdout, stderr=sys.stderr): + """ + Run a Tahoe-LAFS node. + + :param configFactory: A zero-argument callable which creates the config + object to use to parse the argument list. + + :param [str] argv: The argument list to use to configure the run. + + :param stdout: The file-like object to use for stdout. + :param stderr: The file-like object to use for stderr. + + :raise SystemExit: Always raised after the run is complete. + """ # TODO(3035): Remove tox-check when error becomes a warning if 'TOX_ENV_NAME' not in os.environ: assert sys.version_info < (3,), u"Tahoe-LAFS does not run under Python 3. Please use Python 2.7.x." @@ -250,7 +263,23 @@ def _setup_coverage(reactor, argv): def _run_with_reactor(reactor, config, argv, stdout, stderr): + """ + Run a Tahoe-LAFS node using the given reactor. + :param reactor: The reactor to use. This implementation largely ignores + this and lets the rest of the implementation pick its own reactor. + Oops. + + :param twisted.python.usage.Options config: The config object to use to + parse the argument list. + + :param argv: See ``run``. + + :param stdout: See ``run``. + :param stderr: See ``run``. + + :return: A ``Deferred`` that fires when the run is complete. + """ _setup_coverage(reactor, argv) d = defer.maybeDeferred( From 6e152daf05f01191c88a1515f313c060baacdee8 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 17 Dec 2020 09:37:45 -0500 Subject: [PATCH 013/269] Put the pidfile in the right place Seems we relied on the chdir for that to happen, previously. --- src/allmydata/scripts/tahoe_run.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/allmydata/scripts/tahoe_run.py b/src/allmydata/scripts/tahoe_run.py index 071dfac60..80c867202 100644 --- a/src/allmydata/scripts/tahoe_run.py +++ b/src/allmydata/scripts/tahoe_run.py @@ -202,7 +202,9 @@ def run(config, runApp=twistd.runApp): if not nodetype: print("%s is not a recognizable node directory" % quoted_basedir, file=err) return 1 - twistd_args = ["--nodaemon", "--rundir", basedir] + + pidfile = get_pidfile(basedir) + twistd_args = ["--nodaemon", "--rundir", basedir, "--pidfile", pidfile] twistd_args.extend(config.twistd_args) twistd_args.append("DaemonizeTahoeNode") # point at our DaemonizeTahoeNodePlugin @@ -219,7 +221,6 @@ def run(config, runApp=twistd.runApp): twistd_config.loadedPlugins = {"DaemonizeTahoeNode": DaemonizeTahoeNodePlugin(nodetype, basedir)} # handle invalid PID file (twistd might not start otherwise) - pidfile = get_pidfile(basedir) if get_pid_from_pidfile(pidfile) == -1: print("found invalid PID file in %s - deleting it" % basedir, file=err) os.remove(pidfile) From c9b3ccedb4e91b728c449ab504eddabc197f410a Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 4 Jan 2021 11:59:58 -0500 Subject: [PATCH 014/269] explain this while loop --- src/allmydata/scripts/runner.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/allmydata/scripts/runner.py b/src/allmydata/scripts/runner.py index 5b8e2bd4a..9e6aa0bcc 100644 --- a/src/allmydata/scripts/runner.py +++ b/src/allmydata/scripts/runner.py @@ -113,7 +113,7 @@ def parse_or_exit_with_explanation_with_config(config, argv, stdout, stderr): If they are invalid, write an explanation to ``stdout`` and exit. - :param twisted.python.usage.Options config: An instance of the + :param allmydata.scripts.runner.Options config: An instance of the argument-parsing class to use. :param [str] argv: The argument list to parse, including the name of the @@ -129,6 +129,17 @@ def parse_or_exit_with_explanation_with_config(config, argv, stdout, stderr): try: parse_options(argv[1:], config=config) except usage.error as e: + # `parse_options` may have the side-effect of initializing a + # "sub-option" of the given configuration, even if it ultimately + # raises an exception. For example, `tahoe run --invalid-option` will + # set `config.subOptions` to an instance of + # `allmydata.scripts.tahoe_run.RunOptions` and then raise a + # `usage.error` because `RunOptions` does not recognize + # `--invalid-option`. If `run` itself had a sub-options then the same + # thing could happen but with another layer of nesting. We can + # present the user with the most precise information about their usage + # error possible by finding the most "sub" of the sub-options and then + # showing that to the user along with the usage error. c = config while hasattr(c, 'subOptions'): c = c.subOptions From 9958236c3192bda57b8f7f08866f3b2a0b2ba582 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 4 Jan 2021 12:06:03 -0500 Subject: [PATCH 015/269] explain the extra coverage stuff --- src/allmydata/scripts/runner.py | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/src/allmydata/scripts/runner.py b/src/allmydata/scripts/runner.py index 9e6aa0bcc..6d3696d9b 100644 --- a/src/allmydata/scripts/runner.py +++ b/src/allmydata/scripts/runner.py @@ -235,11 +235,28 @@ def run(configFactory=Options, argv=sys.argv, stdout=sys.stdout, stderr=sys.stde def _setup_coverage(reactor, argv): """ - Arrange for coverage to be collected if the 'coverage' package is - installed + If coverage measurement was requested, start collecting coverage + measurements and arrange to record those measurements when the process is + done. + + Coverage measurement is considered requested if ``"--coverage"`` is in + ``argv`` (and it will be removed from ``argv`` if it is found). There + should be a ``.coveragerc`` file in the working directory if coverage + measurement is requested. + + This is only necessary to support multi-process coverage measurement, + typically when the test suite is running, and with the pytest-based + *integration* test suite (at ``integration/`` in the root of the source + tree) foremost in mind. The idea is that if you are running Tahoe-LAFS in + a configuration where multiple processes are involved - for example, a + test process and a client node process, if you only measure coverage from + the test process then you will fail to observe most Tahoe-LAFS code that + is being run. + + This function arranges to have any Tahoe-LAFS process (such as that + client node process) collect and report coverage measurements as well. """ - # can we put this _setup_coverage call after we hit - # argument-parsing? + # can we put this _setup_coverage call after we hit argument-parsing? if '--coverage' not in argv: return argv.remove('--coverage') From 190d9a7319e155ee72918e1cda714e327b27ddc4 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 4 Jan 2021 12:13:53 -0500 Subject: [PATCH 016/269] Skip the pidfile test on Windows where there are no pidfiles --- src/allmydata/test/cli/test_run.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/allmydata/test/cli/test_run.py b/src/allmydata/test/cli/test_run.py index e4bcdfbcd..84befafc1 100644 --- a/src/allmydata/test/cli/test_run.py +++ b/src/allmydata/test/cli/test_run.py @@ -6,12 +6,19 @@ from six.moves import ( StringIO, ) +from testtools import ( + skipIf, +) + from testtools.matchers import ( Contains, Equals, HasLength, ) +from twisted.python.runtime import ( + platform, +) from twisted.python.filepath import ( FilePath, ) @@ -134,6 +141,7 @@ class RunTests(SyncTestCase): """ Tests for ``run``. """ + @skipIf(platform.isWindows(), "There are no PID files on Windows.") def test_non_numeric_pid(self): """ If the pidfile exists but does not contain a numeric value, a complaint to From 916ddd590ea58f48811780d3adf8e8233f4feee5 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 4 Jan 2021 14:13:34 -0500 Subject: [PATCH 017/269] Maybe a useful test to demonstrate the lower-level behavior? Or maybe trash. I don't know. --- src/allmydata/test/test_runner.py | 34 +++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/src/allmydata/test/test_runner.py b/src/allmydata/test/test_runner.py index ef2b99a19..2c8e65fff 100644 --- a/src/allmydata/test/test_runner.py +++ b/src/allmydata/test/test_runner.py @@ -6,6 +6,12 @@ from __future__ import ( import os.path, re, sys from os import linesep +import six + +from testtools import ( + skipUnless, +) + from twisted.trial import unittest from twisted.internet import reactor @@ -22,6 +28,10 @@ from allmydata.util import fileutil, pollmixin from allmydata.util.encodingutil import unicode_to_argv, unicode_to_output from allmydata.test import common_util import allmydata +from allmydata.scripts.runner import ( + parse_options, +) + from .common_util import parse_cli, run_cli from .cli_node_api import ( CLINodeAPI, @@ -36,6 +46,9 @@ from ..util.eliotutil import ( inline_callbacks, log_call_deferred, ) +from .common import ( + SyncTestCase, +) def get_root_from_file(src): srcdir = os.path.dirname(os.path.dirname(os.path.normcase(os.path.realpath(src)))) @@ -72,6 +85,27 @@ class RunBinTahoeMixin(object): return d +class ParseOptionsTests(SyncTestCase): + """ + Tests for ``parse_options``. + """ + @skipUnless(six.PY2, "Only Python 2 exceptions must stringify to bytes.") + def test_nonascii_unknown_subcommand_python2(self): + """ + When ``parse_options`` is called with an argv indicating a subcommand that + does not exist and which also contains non-ascii characters, the + exception it raises includes the subcommand encoded as UTF-8. + """ + tricky = u"\u2621" + try: + parse_options([unicode_to_argv(tricky, mangle=True)]) + except usage.error as e: + self.assertEqual( + b"Unknown command: " + tricky.encode("utf-8"), + str(e) + ) + + class BinTahoe(common_util.SignalMixin, unittest.TestCase, RunBinTahoeMixin): def test_unicode_arguments_and_output(self): tricky = u"\u2621" From 680a5a0575adbb237e050478bfffaaa0452b3878 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 12 Feb 2021 14:26:08 -0500 Subject: [PATCH 018/269] mangling no longer a thing --- src/allmydata/test/test_runner.py | 2 +- src/allmydata/util/encodingutil.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/test_runner.py b/src/allmydata/test/test_runner.py index 6ed9a65c6..0a71f45a3 100644 --- a/src/allmydata/test/test_runner.py +++ b/src/allmydata/test/test_runner.py @@ -86,7 +86,7 @@ class ParseOptionsTests(SyncTestCase): """ tricky = u"\u2621" try: - parse_options([unicode_to_argv(tricky, mangle=True)]) + parse_options([unicode_to_argv(tricky)]) except usage.error as e: self.assertEqual( b"Unknown command: " + tricky.encode("utf-8"), diff --git a/src/allmydata/util/encodingutil.py b/src/allmydata/util/encodingutil.py index 483871b5d..4e1cf57ba 100644 --- a/src/allmydata/util/encodingutil.py +++ b/src/allmydata/util/encodingutil.py @@ -126,7 +126,7 @@ def argv_to_abspath(s, **kwargs): return abspath_expanduser_unicode(decoded, **kwargs) -def unicode_to_argv(s, mangle=False): +def unicode_to_argv(s): """ Make the given unicode string suitable for use in an argv list. From 4575deb27cb320ed22614542fb4b46a4e808c852 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 12 Feb 2021 14:29:04 -0500 Subject: [PATCH 019/269] Attempt to address non-ascii exceptions from the option parser --- src/allmydata/scripts/runner.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/allmydata/scripts/runner.py b/src/allmydata/scripts/runner.py index e83b2b38d..1f326a461 100644 --- a/src/allmydata/scripts/runner.py +++ b/src/allmydata/scripts/runner.py @@ -108,7 +108,17 @@ for module in (create_node,): def parse_options(argv, config=None): if not config: config = Options() - config.parseOptions(argv) # may raise usage.error + try: + config.parseOptions(argv) + except usage.error as e: + if six.PY2: + # Exceptions must stringify to bytes on Python 2. + raise usage.error(*( + arg.encode("utf-8") if isinstance(arg, unicode) else arg + for arg + in e.args + )) + raise return config def parse_or_exit_with_explanation_with_config(config, argv, stdout, stderr): From 50e033f263cf85c91ed9ba807d700e7f31c5337f Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 12 Feb 2021 14:37:53 -0500 Subject: [PATCH 020/269] Log unhandled output from the tahoe runner helper --- src/allmydata/test/cli_node_api.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/allmydata/test/cli_node_api.py b/src/allmydata/test/cli_node_api.py index 34d73a199..aaa280371 100644 --- a/src/allmydata/test/cli_node_api.py +++ b/src/allmydata/test/cli_node_api.py @@ -24,6 +24,9 @@ from twisted.internet.error import ( from twisted.internet.interfaces import ( IProcessProtocol, ) +from twisted.python.log import ( + msg, +) from twisted.python.filepath import ( FilePath, ) @@ -88,7 +91,10 @@ class _ProcessProtocolAdapter(ProcessProtocol, object): try: proto = self._fds[childFD] except KeyError: - pass + msg("Received unhandled output on {fd}: {output}", + fd=childFD, + output=data, + ) else: proto.dataReceived(data) From 6458183df2531a3e80f3e62baf8080ba77317664 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 12 Feb 2021 14:38:43 -0500 Subject: [PATCH 021/269] maybe it's this --- src/allmydata/test/cli_node_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/cli_node_api.py b/src/allmydata/test/cli_node_api.py index aaa280371..fbfa55287 100644 --- a/src/allmydata/test/cli_node_api.py +++ b/src/allmydata/test/cli_node_api.py @@ -91,7 +91,7 @@ class _ProcessProtocolAdapter(ProcessProtocol, object): try: proto = self._fds[childFD] except KeyError: - msg("Received unhandled output on {fd}: {output}", + msg("Received unhandled output on %(fd)s: %(output)s", fd=childFD, output=data, ) From 47b60c0faa0ba0304b241332914e0dbba337b272 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 12 Feb 2021 14:39:28 -0500 Subject: [PATCH 022/269] oh yea I think it's this --- src/allmydata/test/cli_node_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/cli_node_api.py b/src/allmydata/test/cli_node_api.py index fbfa55287..f43fce148 100644 --- a/src/allmydata/test/cli_node_api.py +++ b/src/allmydata/test/cli_node_api.py @@ -91,7 +91,7 @@ class _ProcessProtocolAdapter(ProcessProtocol, object): try: proto = self._fds[childFD] except KeyError: - msg("Received unhandled output on %(fd)s: %(output)s", + msg(format="Received unhandled output on %(fd)s: %(output)s", fd=childFD, output=data, ) From e9adccd43238ec900435ae3e03854ff8f01ad7e5 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 12 Feb 2021 14:42:30 -0500 Subject: [PATCH 023/269] more logs --- src/allmydata/test/cli_node_api.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/allmydata/test/cli_node_api.py b/src/allmydata/test/cli_node_api.py index f43fce148..a5c877b0b 100644 --- a/src/allmydata/test/cli_node_api.py +++ b/src/allmydata/test/cli_node_api.py @@ -152,6 +152,9 @@ class CLINodeAPI(object): u"-m", u"allmydata.scripts.runner", ] + argv + msg(format="Executing %(argv)s", + argv=argv, + ) return self.reactor.spawnProcess( processProtocol=process_protocol, executable=exe, From f0ac092109388c49c70a5586e1680e5199dc420f Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 12 Feb 2021 14:49:20 -0500 Subject: [PATCH 024/269] Avoid the pidfile stuff on Windows --- src/allmydata/scripts/tahoe_run.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/allmydata/scripts/tahoe_run.py b/src/allmydata/scripts/tahoe_run.py index 80c867202..a1524ab35 100644 --- a/src/allmydata/scripts/tahoe_run.py +++ b/src/allmydata/scripts/tahoe_run.py @@ -203,8 +203,10 @@ def run(config, runApp=twistd.runApp): print("%s is not a recognizable node directory" % quoted_basedir, file=err) return 1 - pidfile = get_pidfile(basedir) - twistd_args = ["--nodaemon", "--rundir", basedir, "--pidfile", pidfile] + twistd_args = ["--nodaemon", "--rundir", basedir] + if sys.platform != "win32": + pidfile = get_pidfile(basedir) + twistd_args.extend(["--pidfile", pidfile]) twistd_args.extend(config.twistd_args) twistd_args.append("DaemonizeTahoeNode") # point at our DaemonizeTahoeNodePlugin @@ -221,7 +223,7 @@ def run(config, runApp=twistd.runApp): twistd_config.loadedPlugins = {"DaemonizeTahoeNode": DaemonizeTahoeNodePlugin(nodetype, basedir)} # handle invalid PID file (twistd might not start otherwise) - if get_pid_from_pidfile(pidfile) == -1: + if sys.platform != "win32" and get_pid_from_pidfile(pidfile) == -1: print("found invalid PID file in %s - deleting it" % basedir, file=err) os.remove(pidfile) From 7396130c0a6cc3b9fa05765bf885a8d103915360 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 27 Jul 2021 14:20:01 -0400 Subject: [PATCH 025/269] Integration test for I2P. --- integration/test_i2p.py | 239 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 239 insertions(+) create mode 100644 integration/test_i2p.py diff --git a/integration/test_i2p.py b/integration/test_i2p.py new file mode 100644 index 000000000..e87ba28e2 --- /dev/null +++ b/integration/test_i2p.py @@ -0,0 +1,239 @@ +""" +Integration tests for I2P support. +""" + +from __future__ import unicode_literals +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + +import sys +from os.path import join, exists +from os import mkdir +from time import sleep + +if PY2: + def which(path): + # This will result in skipping I2P tests on Python 2. Oh well. + return None +else: + from shutil import which + +from eliot import log_call + +import pytest +import pytest_twisted + +from . import util + +from twisted.python.filepath import ( + FilePath, +) +from twisted.internet.error import ProcessExitedAlready + +from allmydata.test.common import ( + write_introducer, +) + +if which("docker") is None: + pytest.skip('Skipping I2P tests since Docker is unavailable', allow_module_level=True) + + +@pytest.fixture(scope="session") +def i2p_network(reactor, temp_dir, request): + """Fixture to start up local i2pd.""" + proto = util._MagicTextProtocol("ephemeral keys") + reactor.spawnProcess( + proto, + which("docker"), + ( + "docker", "run", "-p", "7656:7656", "purplei2p/i2pd", + # Bad URL for reseeds, so it can't talk to other routers. + "--reseed.urls", "http://localhost:1/", + ), + ) + pytest_twisted.blockon(proto.magic_seen) + + def cleanup(): + try: + proto.transport.signalProcess("INT") + util.block_with_timeout(proto.exited, reactor) + except ProcessExitedAlready: + pass + request.addfinalizer(cleanup) + + +@pytest.fixture(scope='session') +@log_call( + action_type=u"integration:i2p:introducer", + include_args=["temp_dir", "flog_gatherer"], + include_result=False, +) +def i2p_introducer(reactor, temp_dir, flog_gatherer, request): + config = ''' +[node] +nickname = introducer_i2p +web.port = 4561 +log_gatherer.furl = {log_furl} +'''.format(log_furl=flog_gatherer) + + intro_dir = join(temp_dir, 'introducer_i2p') + print("making introducer", intro_dir) + + if not exists(intro_dir): + mkdir(intro_dir) + done_proto = util._ProcessExitedProtocol() + util._tahoe_runner_optional_coverage( + done_proto, + reactor, + request, + ( + 'create-introducer', + '--listen=i2p', + intro_dir, + ), + ) + pytest_twisted.blockon(done_proto.done) + + # over-write the config file with our stuff + with open(join(intro_dir, 'tahoe.cfg'), 'w') as f: + f.write(config) + + # "tahoe run" is consistent across Linux/macOS/Windows, unlike the old + # "start" command. + protocol = util._MagicTextProtocol('introducer running') + transport = util._tahoe_runner_optional_coverage( + protocol, + reactor, + request, + ( + 'run', + intro_dir, + ), + ) + + def cleanup(): + try: + transport.signalProcess('TERM') + util.block_with_timeout(protocol.exited, reactor) + except ProcessExitedAlready: + pass + request.addfinalizer(cleanup) + + pytest_twisted.blockon(protocol.magic_seen) + return transport + + +@pytest.fixture(scope='session') +def i2p_introducer_furl(i2p_introducer, temp_dir): + furl_fname = join(temp_dir, 'introducer_i2p', 'private', 'introducer.furl') + while not exists(furl_fname): + print("Don't see {} yet".format(furl_fname)) + sleep(.1) + furl = open(furl_fname, 'r').read() + return furl + + +@pytest_twisted.inlineCallbacks +def test_i2p_service_storage(reactor, request, temp_dir, flog_gatherer, i2p_network, i2p_introducer_furl): + yield _create_anonymous_node(reactor, 'carol', 8008, request, temp_dir, flog_gatherer, i2p_network, i2p_introducer_furl) + yield _create_anonymous_node(reactor, 'dave', 8009, request, temp_dir, flog_gatherer, i2p_network, i2p_introducer_furl) + # ensure both nodes are connected to "a grid" by uploading + # something via carol, and retrieve it using dave. + gold_path = join(temp_dir, "gold") + with open(gold_path, "w") as f: + f.write( + "The object-capability model is a computer security model. A " + "capability describes a transferable right to perform one (or " + "more) operations on a given object." + ) + # XXX could use treq or similar to POST these to their respective + # WUIs instead ... + + proto = util._CollectOutputProtocol() + reactor.spawnProcess( + proto, + sys.executable, + ( + sys.executable, '-b', '-m', 'allmydata.scripts.runner', + '-d', join(temp_dir, 'carol'), + 'put', gold_path, + ) + ) + yield proto.done + cap = proto.output.getvalue().strip().split()[-1] + print("TEH CAP!", cap) + + proto = util._CollectOutputProtocol(capture_stderr=False) + reactor.spawnProcess( + proto, + sys.executable, + ( + sys.executable, '-b', '-m', 'allmydata.scripts.runner', + '-d', join(temp_dir, 'dave'), + 'get', cap, + ) + ) + yield proto.done + + dave_got = proto.output.getvalue().strip() + assert dave_got == open(gold_path, 'rb').read().strip() + + +@pytest_twisted.inlineCallbacks +def _create_anonymous_node(reactor, name, control_port, request, temp_dir, flog_gatherer, i2p_network, introducer_furl): + node_dir = FilePath(temp_dir).child(name) + web_port = "tcp:{}:interface=localhost".format(control_port + 2000) + + print("creating", node_dir.path) + node_dir.makedirs() + proto = util._DumpOutputProtocol(None) + reactor.spawnProcess( + proto, + sys.executable, + ( + sys.executable, '-b', '-m', 'allmydata.scripts.runner', + 'create-node', + '--nickname', name, + '--introducer', introducer_furl, + '--hide-ip', + '--listen', 'i2p', + node_dir.path, + ) + ) + yield proto.done + + + # Which services should this client connect to? + write_introducer(node_dir, "default", introducer_furl) + with node_dir.child('tahoe.cfg').open('w') as f: + node_config = ''' +[node] +nickname = %(name)s +web.port = %(web_port)s +web.static = public_html +log_gatherer.furl = %(log_furl)s + +[i2p] +enabled = true + +[client] +shares.needed = 1 +shares.happy = 1 +shares.total = 2 + +''' % { + 'name': name, + 'web_port': web_port, + 'log_furl': flog_gatherer, +} + node_config = node_config.encode("utf-8") + f.write(node_config) + + print("running") + yield util._run_node(reactor, node_dir.path, request, None) + print("okay, launched") From dbd2e7f9730f47184213bdd926e639e99dea0f0b Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 27 Jul 2021 14:20:26 -0400 Subject: [PATCH 026/269] News file. --- newsfragments/3743.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3743.minor diff --git a/newsfragments/3743.minor b/newsfragments/3743.minor new file mode 100644 index 000000000..e69de29bb From ddca3e9ab8139518e611ff3f7dd7e84e399c1a0e Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 27 Jul 2021 14:21:40 -0400 Subject: [PATCH 027/269] At this point all integration tests are expected to pass on Python 3. --- src/allmydata/util/_python3.py | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py index 8025582f2..f65e0aaf9 100644 --- a/src/allmydata/util/_python3.py +++ b/src/allmydata/util/_python3.py @@ -17,21 +17,6 @@ if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 -PORTED_INTEGRATION_TESTS = [ - "integration.test_aaa_aardvark", - "integration.test_servers_of_happiness", - "integration.test_sftp", - "integration.test_streaming_logs", - "integration.test_tor", - "integration.test_web", -] - -PORTED_INTEGRATION_MODULES = [ - "integration", - "integration.conftest", - "integration.util", -] - # Keep these sorted alphabetically, to reduce merge conflicts: PORTED_MODULES = [ "allmydata", From 982ac3cc33aad62eb647738245a3b9e52882c26f Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 29 Jul 2021 10:02:02 -0400 Subject: [PATCH 028/269] Timeout if i2pd never starts. --- integration/test_i2p.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/integration/test_i2p.py b/integration/test_i2p.py index e87ba28e2..3f2133f6f 100644 --- a/integration/test_i2p.py +++ b/integration/test_i2p.py @@ -56,7 +56,6 @@ def i2p_network(reactor, temp_dir, request): "--reseed.urls", "http://localhost:1/", ), ) - pytest_twisted.blockon(proto.magic_seen) def cleanup(): try: @@ -66,6 +65,8 @@ def i2p_network(reactor, temp_dir, request): pass request.addfinalizer(cleanup) + util.block_with_timeout(proto.magic_seen, reactor, timeout=30) + @pytest.fixture(scope='session') @log_call( From 97522641d669662971ce29fb7678a4b6eec50d8d Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 30 Jul 2021 11:06:28 -0400 Subject: [PATCH 029/269] Skip on Windows. --- integration/test_i2p.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/integration/test_i2p.py b/integration/test_i2p.py index 3f2133f6f..4f212f5fb 100644 --- a/integration/test_i2p.py +++ b/integration/test_i2p.py @@ -41,6 +41,10 @@ from allmydata.test.common import ( if which("docker") is None: pytest.skip('Skipping I2P tests since Docker is unavailable', allow_module_level=True) +# Docker on Windows machines sometimes expects Windows-y Docker images, so just +# don't bother. +if sys.platform.startswith('win'): + pytest.skip('Skipping I2P tests on Windows', allow_module_level=True) @pytest.fixture(scope="session") From ce2363e3ded009bc3604a88af6445903dd45ea81 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 30 Jul 2021 11:08:56 -0400 Subject: [PATCH 030/269] More aggressively shut down i2pd and other i2p-related processes. --- integration/test_i2p.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/integration/test_i2p.py b/integration/test_i2p.py index 4f212f5fb..1aa7a5d99 100644 --- a/integration/test_i2p.py +++ b/integration/test_i2p.py @@ -47,7 +47,7 @@ if sys.platform.startswith('win'): pytest.skip('Skipping I2P tests on Windows', allow_module_level=True) -@pytest.fixture(scope="session") +@pytest.fixture def i2p_network(reactor, temp_dir, request): """Fixture to start up local i2pd.""" proto = util._MagicTextProtocol("ephemeral keys") @@ -63,7 +63,7 @@ def i2p_network(reactor, temp_dir, request): def cleanup(): try: - proto.transport.signalProcess("INT") + proto.transport.signalProcess("KILL") util.block_with_timeout(proto.exited, reactor) except ProcessExitedAlready: pass @@ -72,7 +72,7 @@ def i2p_network(reactor, temp_dir, request): util.block_with_timeout(proto.magic_seen, reactor, timeout=30) -@pytest.fixture(scope='session') +@pytest.fixture @log_call( action_type=u"integration:i2p:introducer", include_args=["temp_dir", "flog_gatherer"], @@ -133,7 +133,7 @@ log_gatherer.furl = {log_furl} return transport -@pytest.fixture(scope='session') +@pytest.fixture def i2p_introducer_furl(i2p_introducer, temp_dir): furl_fname = join(temp_dir, 'introducer_i2p', 'private', 'introducer.furl') while not exists(furl_fname): From d060af641a553f9feda6124d5beecb05a7115abb Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 30 Jul 2021 11:13:57 -0400 Subject: [PATCH 031/269] Python 3 port is essentially done-get rid of relevant tests and tracking code. --- newsfragments/3751.minor | 0 nix/tahoe-lafs.nix | 4 - src/allmydata/test/test_python3.py | 122 ----------- src/allmydata/util/_python3.py | 311 ----------------------------- 4 files changed, 437 deletions(-) create mode 100644 newsfragments/3751.minor delete mode 100644 src/allmydata/test/test_python3.py delete mode 100644 src/allmydata/util/_python3.py diff --git a/newsfragments/3751.minor b/newsfragments/3751.minor new file mode 100644 index 000000000..e69de29bb diff --git a/nix/tahoe-lafs.nix b/nix/tahoe-lafs.nix index 8005ca50b..35b29f1cc 100644 --- a/nix/tahoe-lafs.nix +++ b/nix/tahoe-lafs.nix @@ -72,10 +72,6 @@ python.pkgs.buildPythonPackage rec { rm src/allmydata/test/test_connections.py rm src/allmydata/test/cli/test_create.py - # Since we're deleting files, this complains they're missing. For now Nix - # is Python 2-only, anyway, so these tests don't add anything yet. - rm src/allmydata/test/test_python3.py - # Generate _version.py ourselves since we can't rely on the Python code # extracting the information from the .git directory we excluded. cat > src/allmydata/_version.py < Date: Fri, 30 Jul 2021 11:27:41 -0400 Subject: [PATCH 032/269] Choose node names that won't conflict. --- integration/test_i2p.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/integration/test_i2p.py b/integration/test_i2p.py index 1aa7a5d99..f0b06f1e2 100644 --- a/integration/test_i2p.py +++ b/integration/test_i2p.py @@ -145,8 +145,8 @@ def i2p_introducer_furl(i2p_introducer, temp_dir): @pytest_twisted.inlineCallbacks def test_i2p_service_storage(reactor, request, temp_dir, flog_gatherer, i2p_network, i2p_introducer_furl): - yield _create_anonymous_node(reactor, 'carol', 8008, request, temp_dir, flog_gatherer, i2p_network, i2p_introducer_furl) - yield _create_anonymous_node(reactor, 'dave', 8009, request, temp_dir, flog_gatherer, i2p_network, i2p_introducer_furl) + yield _create_anonymous_node(reactor, 'carol_i2p', 8008, request, temp_dir, flog_gatherer, i2p_network, i2p_introducer_furl) + yield _create_anonymous_node(reactor, 'dave_i2p', 8009, request, temp_dir, flog_gatherer, i2p_network, i2p_introducer_furl) # ensure both nodes are connected to "a grid" by uploading # something via carol, and retrieve it using dave. gold_path = join(temp_dir, "gold") @@ -165,7 +165,7 @@ def test_i2p_service_storage(reactor, request, temp_dir, flog_gatherer, i2p_netw sys.executable, ( sys.executable, '-b', '-m', 'allmydata.scripts.runner', - '-d', join(temp_dir, 'carol'), + '-d', join(temp_dir, 'carol_i2p'), 'put', gold_path, ) ) @@ -179,7 +179,7 @@ def test_i2p_service_storage(reactor, request, temp_dir, flog_gatherer, i2p_netw sys.executable, ( sys.executable, '-b', '-m', 'allmydata.scripts.runner', - '-d', join(temp_dir, 'dave'), + '-d', join(temp_dir, 'dave_i2p'), 'get', cap, ) ) From 90e84730e57e8cba179889c4242a34452d6a064e Mon Sep 17 00:00:00 2001 From: fenn-cs Date: Sun, 8 Aug 2021 21:49:02 +0100 Subject: [PATCH 033/269] Fixes 3757 : Refactored test_introducer in web tests to use custom base test cases Signed-off-by: fenn-cs --- newsfragments/3757.other | 0 src/allmydata/test/web/test_introducer.py | 33 ++++++++++++++--------- 2 files changed, 20 insertions(+), 13 deletions(-) create mode 100644 newsfragments/3757.other diff --git a/newsfragments/3757.other b/newsfragments/3757.other new file mode 100644 index 000000000..e69de29bb diff --git a/src/allmydata/test/web/test_introducer.py b/src/allmydata/test/web/test_introducer.py index 08d95bda9..34b1088e3 100644 --- a/src/allmydata/test/web/test_introducer.py +++ b/src/allmydata/test/web/test_introducer.py @@ -18,6 +18,12 @@ from bs4 import BeautifulSoup from twisted.trial import unittest from twisted.internet import reactor from twisted.internet import defer +from testtools.twistedsupport import succeeded + +from ..common import ( + SyncTestCase, + AsyncTestCase, +) from foolscap.api import ( fireEventually, @@ -53,6 +59,10 @@ from ..common_web import ( render, ) +from testtools.matchers import ( + Equals +) + @defer.inlineCallbacks def create_introducer_webish(reactor, port_assigner, basedir): @@ -86,11 +96,10 @@ def create_introducer_webish(reactor, port_assigner, basedir): yield fireEventually(None) intro_node.startService() - defer.returnValue((intro_node, ws)) -class IntroducerWeb(unittest.TestCase): +class IntroducerWeb(AsyncTestCase): """ Tests for web-facing functionality of an introducer node. """ @@ -102,6 +111,8 @@ class IntroducerWeb(unittest.TestCase): # Anything using Foolscap leaves some timer trash in the reactor that # we have to arrange to have cleaned up. self.addCleanup(lambda: flushEventualQueue(None)) + return super(IntroducerWeb, self).setUp() + @defer.inlineCallbacks def test_welcome(self): @@ -187,7 +198,7 @@ class IntroducerWeb(unittest.TestCase): self.assertEqual(data["announcement_summary"], {}) -class IntroducerRootTests(unittest.TestCase): +class IntroducerRootTests(SyncTestCase): """ Tests for ``IntroducerRoot``. """ @@ -222,16 +233,12 @@ class IntroducerRootTests(unittest.TestCase): 0, ) - resource = IntroducerRoot(introducer_node) - response = json.loads( - self.successResultOf( + resource = IntroducerRoot(introducer_node) + response = json.loads(succeeded( render(resource, {b"t": [b"json"]}), - ), - ) - self.assertEqual( - response, - { + )._matcher.result) + deferred = defer.succeed({ u"subscription_summary": {"arbitrary": 2}, u"announcement_summary": {"arbitrary": 1}, - }, - ) + }) + self.assertThat(deferred.result, Equals(response)) From 7ad3fa9e25975eb749345af7ea9745dfe166f7e7 Mon Sep 17 00:00:00 2001 From: fenn-cs Date: Mon, 9 Aug 2021 23:46:30 +0100 Subject: [PATCH 034/269] correct use of succeeded, remove trailing spaces and unused import Signed-off-by: fenn-cs --- src/allmydata/test/web/test_introducer.py | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/src/allmydata/test/web/test_introducer.py b/src/allmydata/test/web/test_introducer.py index 34b1088e3..b1bc69f17 100644 --- a/src/allmydata/test/web/test_introducer.py +++ b/src/allmydata/test/web/test_introducer.py @@ -15,7 +15,6 @@ from os.path import join from bs4 import BeautifulSoup -from twisted.trial import unittest from twisted.internet import reactor from twisted.internet import defer from testtools.twistedsupport import succeeded @@ -60,7 +59,8 @@ from ..common_web import ( ) from testtools.matchers import ( - Equals + Equals, + AfterPreprocessing, ) @@ -112,7 +112,6 @@ class IntroducerWeb(AsyncTestCase): # we have to arrange to have cleaned up. self.addCleanup(lambda: flushEventualQueue(None)) return super(IntroducerWeb, self).setUp() - @defer.inlineCallbacks def test_welcome(self): @@ -233,12 +232,10 @@ class IntroducerRootTests(SyncTestCase): 0, ) - resource = IntroducerRoot(introducer_node) - response = json.loads(succeeded( - render(resource, {b"t": [b"json"]}), - )._matcher.result) - deferred = defer.succeed({ - u"subscription_summary": {"arbitrary": 2}, - u"announcement_summary": {"arbitrary": 1}, - }) - self.assertThat(deferred.result, Equals(response)) + resource = IntroducerRoot(introducer_node) + response = render(resource, {b"t": [b"json"]}) + expected = { + u"subscription_summary": {"arbitrary": 2}, + u"announcement_summary": {"arbitrary": 1}, + } + self.assertThat(response, succeeded(AfterPreprocessing(json.loads, Equals(expected))) From a6b7c07e1cbbfbc6f1aa00a33dd64e1c10159d97 Mon Sep 17 00:00:00 2001 From: fenn-cs Date: Tue, 10 Aug 2021 10:13:00 +0100 Subject: [PATCH 035/269] added missing parathensis, ran yapf Signed-off-by: fenn-cs --- src/allmydata/test/web/test_introducer.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/allmydata/test/web/test_introducer.py b/src/allmydata/test/web/test_introducer.py index b1bc69f17..ba0a5beb9 100644 --- a/src/allmydata/test/web/test_introducer.py +++ b/src/allmydata/test/web/test_introducer.py @@ -238,4 +238,6 @@ class IntroducerRootTests(SyncTestCase): u"subscription_summary": {"arbitrary": 2}, u"announcement_summary": {"arbitrary": 1}, } - self.assertThat(response, succeeded(AfterPreprocessing(json.loads, Equals(expected))) + self.assertThat( + response, + succeeded(AfterPreprocessing(json.loads, Equals(expected)))) From 78a3ca52c4b2eb0e891b4c50d7461eee46214486 Mon Sep 17 00:00:00 2001 From: fenn-cs Date: Tue, 10 Aug 2021 19:00:05 +0100 Subject: [PATCH 036/269] DUMMY COMMIT TO TRIGGER CI : added commit text to fragments Signed-off-by: fenn-cs --- newsfragments/3757.other | 1 + 1 file changed, 1 insertion(+) diff --git a/newsfragments/3757.other b/newsfragments/3757.other index e69de29bb..3d2d3f272 100644 --- a/newsfragments/3757.other +++ b/newsfragments/3757.other @@ -0,0 +1 @@ +Refactored test_introducer in web tests to use custom base test cases \ No newline at end of file From a4da8048700204f50e377a44e8116e519fcffde7 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 11 Aug 2021 07:33:58 -0400 Subject: [PATCH 037/269] Stop using the dockerhub-auth context for normal jobs --- .circleci/config.yml | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1fb7558de..8aa039694 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -16,18 +16,8 @@ workflows: jobs: # Start with jobs testing various platforms. - # Every job that pulls a Docker image from Docker Hub needs to provide - # credentials for that pull operation to avoid being subjected to - # unauthenticated pull limits shared across all of CircleCI. Use this - # first job to define a yaml anchor that can be used to supply a - # CircleCI job context which makes Docker Hub credentials available in - # the environment. - # - # Contexts are managed in the CircleCI web interface: - # - # https://app.circleci.com/settings/organization/github/tahoe-lafs/contexts - "debian-9": &DOCKERHUB_CONTEXT - context: "dockerhub-auth" + {} - "debian-10": <<: *DOCKERHUB_CONTEXT From c5fec82328231c718ce8bcfbf02c670bf3305136 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 11 Aug 2021 07:40:16 -0400 Subject: [PATCH 038/269] CircleCI succeeds in pulling the docker images without this It says "Warning!" but then says it is going to use its own credentials. Great. Just what I want. --- .circleci/config.yml | 25 +------------------------ 1 file changed, 1 insertion(+), 24 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 8aa039694..3e48203c0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -15,72 +15,49 @@ workflows: ci: jobs: # Start with jobs testing various platforms. - - - "debian-9": &DOCKERHUB_CONTEXT - {} - + - "debian-9": - "debian-10": - <<: *DOCKERHUB_CONTEXT requires: - "debian-9" - "ubuntu-20-04": - <<: *DOCKERHUB_CONTEXT - "ubuntu-18-04": - <<: *DOCKERHUB_CONTEXT requires: - "ubuntu-20-04" - "ubuntu-16-04": - <<: *DOCKERHUB_CONTEXT requires: - "ubuntu-20-04" - "fedora-29": - <<: *DOCKERHUB_CONTEXT - "fedora-28": - <<: *DOCKERHUB_CONTEXT requires: - "fedora-29" - "centos-8": - <<: *DOCKERHUB_CONTEXT - - "nixos-19-09": - <<: *DOCKERHUB_CONTEXT # Test against PyPy 2.7 - "pypy27-buster": - <<: *DOCKERHUB_CONTEXT # Just one Python 3.6 configuration while the port is in-progress. - "python36": - <<: *DOCKERHUB_CONTEXT # Other assorted tasks and configurations - "lint": - <<: *DOCKERHUB_CONTEXT - "pyinstaller": - <<: *DOCKERHUB_CONTEXT - "deprecations": - <<: *DOCKERHUB_CONTEXT - "c-locale": - <<: *DOCKERHUB_CONTEXT # Any locale other than C or UTF-8. - "another-locale": - <<: *DOCKERHUB_CONTEXT - "integration": - <<: *DOCKERHUB_CONTEXT requires: # If the unit test suite doesn't pass, don't bother running the # integration tests. - "debian-9" - "typechecks": - <<: *DOCKERHUB_CONTEXT - - "docs": - <<: *DOCKERHUB_CONTEXT images: # Build the Docker images used by the ci jobs. This makes the ci jobs From 6f36f85a87330569475eb81cb483d600107730eb Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 11 Aug 2021 07:41:59 -0400 Subject: [PATCH 039/269] Define the yaml anchor elsewhere, where it is still needed --- .circleci/config.yml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3e48203c0..7db863847 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -72,8 +72,16 @@ workflows: - "master" jobs: - - "build-image-debian-10": - <<: *DOCKERHUB_CONTEXT + # Every job that pushes a Docker image from Docker Hub needs to provide + # credentials. Use this first job to define a yaml anchor that can be + # used to supply a CircleCI job context which makes Docker Hub + # credentials available in the environment. + # + # Contexts are managed in the CircleCI web interface: + # + # https://app.circleci.com/settings/organization/github/tahoe-lafs/contexts + - "build-image-debian-10": &DOCKERHUB_CONTEXT + context: "dockerhub-auth" - "build-image-debian-9": <<: *DOCKERHUB_CONTEXT - "build-image-ubuntu-16-04": From 0e2c4ff7e65312a420df2c6b0effb939caa86c17 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 11 Aug 2021 07:45:45 -0400 Subject: [PATCH 040/269] I suppose you cannot define an empty yaml map by leaving everything out --- .circleci/config.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 7db863847..f2d5a29b9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -16,11 +16,13 @@ workflows: jobs: # Start with jobs testing various platforms. - "debian-9": + {} - "debian-10": requires: - "debian-9" - "ubuntu-20-04": + {} - "ubuntu-18-04": requires: - "ubuntu-20-04" @@ -29,26 +31,37 @@ workflows: - "ubuntu-20-04" - "fedora-29": + {} - "fedora-28": requires: - "fedora-29" - "centos-8": + {} + - "nixos-19-09": + {} # Test against PyPy 2.7 - "pypy27-buster": + {} # Just one Python 3.6 configuration while the port is in-progress. - "python36": + {} # Other assorted tasks and configurations - "lint": + {} - "pyinstaller": + {} - "deprecations": + {} - "c-locale": + {} # Any locale other than C or UTF-8. - "another-locale": + {} - "integration": requires: @@ -57,7 +70,9 @@ workflows: - "debian-9" - "typechecks": + {} - "docs": + {} images: # Build the Docker images used by the ci jobs. This makes the ci jobs From 2bb310c5110518143a6358f3cf1b8685e811421b Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 11 Aug 2021 09:50:23 -0400 Subject: [PATCH 041/269] Try to run codechecks3 on CI --- .circleci/config.yml | 20 +++++++++++++++++++- newsfragments/3760.minor | 0 2 files changed, 19 insertions(+), 1 deletion(-) create mode 100644 newsfragments/3760.minor diff --git a/.circleci/config.yml b/.circleci/config.yml index 1fb7558de..153b29089 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -150,7 +150,7 @@ jobs: # Since this job is never scheduled this step is never run so the # actual value here is irrelevant. - lint: + codechecks: docker: - <<: *DOCKERHUB_AUTH image: "circleci/python:2" @@ -168,6 +168,24 @@ jobs: command: | ~/.local/bin/tox -e codechecks + codechecks3: + docker: + - <<: *DOCKERHUB_AUTH + image: "circleci/python:3" + + steps: + - "checkout" + + - run: + name: "Install tox" + command: | + pip install --user tox + + - run: + name: "Static-ish code checks" + command: | + ~/.local/bin/tox -e codechecks3 + pyinstaller: docker: - <<: *DOCKERHUB_AUTH diff --git a/newsfragments/3760.minor b/newsfragments/3760.minor new file mode 100644 index 000000000..e69de29bb From 02f82e2665b7c8af1ab6a7278435bdfda837ca3d Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 11 Aug 2021 10:00:06 -0400 Subject: [PATCH 042/269] news fragment --- newsfragments/3759.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3759.minor diff --git a/newsfragments/3759.minor b/newsfragments/3759.minor new file mode 100644 index 000000000..e69de29bb From 97008b70b2592c94ee60c4f040a62d526c44f9c2 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 11 Aug 2021 10:25:42 -0400 Subject: [PATCH 043/269] Avoid renaming a job, it causes operational hassle --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 8aa459583..5b123dfa8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -140,7 +140,7 @@ jobs: # Since this job is never scheduled this step is never run so the # actual value here is irrelevant. - codechecks: + lint: docker: - <<: *DOCKERHUB_AUTH image: "circleci/python:2" From a1112e4cd0dbbe1ddfb542d371203b1a0cb846ab Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 11 Aug 2021 10:25:49 -0400 Subject: [PATCH 044/269] Add the new job to the workflow --- .circleci/config.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 5b123dfa8..28e4c8d58 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -53,6 +53,8 @@ workflows: # Other assorted tasks and configurations - "lint": {} + - "codechecks3": + {} - "pyinstaller": {} - "deprecations": From 15312009cecc2355cd1a180b45af6bf760a22de5 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 11 Aug 2021 12:58:51 -0400 Subject: [PATCH 045/269] Fix mis-merge: This test moved to test_run.py --- src/allmydata/test/cli/test_cli.py | 31 +----------------------------- 1 file changed, 1 insertion(+), 30 deletions(-) diff --git a/src/allmydata/test/cli/test_cli.py b/src/allmydata/test/cli/test_cli.py index 7d8ad2046..72eb011d0 100644 --- a/src/allmydata/test/cli/test_cli.py +++ b/src/allmydata/test/cli/test_cli.py @@ -12,13 +12,11 @@ if PY2: from six.moves import cStringIO as StringIO import re -from six import ensure_text, ensure_str +from six import ensure_text import os.path -from mock import patch from urllib.parse import quote as url_quote -from twisted.python.filepath import FilePath from twisted.trial import unittest from twisted.internet.testing import ( MemoryReactor, @@ -1324,30 +1322,3 @@ class Options(ReallyEqualMixin, unittest.TestCase): ["--node-directory=there", "run", some_twistd_option]) self.failUnlessRaises(usage.UsageError, self.parse, ["run", "--basedir=here", some_twistd_option]) - - -class Run(unittest.TestCase): - - @patch('allmydata.scripts.tahoe_run.os.chdir') - @patch('allmydata.scripts.tahoe_run.twistd') - def test_non_numeric_pid(self, mock_twistd, chdir): - """ - If the pidfile exists but does not contain a numeric value, a complaint to - this effect is written to stderr. - """ - basedir = FilePath(ensure_str(self.mktemp())) - basedir.makedirs() - basedir.child(u"twistd.pid").setContent(b"foo") - basedir.child(u"tahoe-client.tac").setContent(b"") - - config = tahoe_run.RunOptions() - config.stdout = StringIO() - config.stderr = StringIO() - config['basedir'] = ensure_text(basedir.path) - config.twistd_args = [] - - result_code = tahoe_run.run(config) - self.assertIn("invalid PID file", config.stderr.getvalue()) - self.assertTrue(len(mock_twistd.mock_calls), 1) - self.assertEqual(mock_twistd.mock_calls[0][0], 'runApp') - self.assertEqual(0, result_code) From 9632b35abebc0e57ae851a8fb2c2a899c9009b4f Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 11 Aug 2021 13:19:15 -0400 Subject: [PATCH 046/269] Fix mismerge: Put Py3 warning back --- src/allmydata/scripts/runner.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/allmydata/scripts/runner.py b/src/allmydata/scripts/runner.py index d48dd1395..ba979fdfa 100644 --- a/src/allmydata/scripts/runner.py +++ b/src/allmydata/scripts/runner.py @@ -221,6 +221,9 @@ def _maybe_enable_eliot_logging(options, reactor): # Pass on the options so we can dispatch the subcommand. return options +PYTHON_3_WARNING = ("Support for Python 3 is an incomplete work-in-progress." + " Use at your own risk.") + def run(configFactory=Options, argv=sys.argv, stdout=sys.stdout, stderr=sys.stderr): """ Run a Tahoe-LAFS node. @@ -235,6 +238,8 @@ def run(configFactory=Options, argv=sys.argv, stdout=sys.stdout, stderr=sys.stde :raise SystemExit: Always raised after the run is complete. """ + if six.PY3: + print(PYTHON_3_WARNING, file=sys.stderr) if sys.platform == "win32": from allmydata.windows.fixups import initialize initialize() From ffbcbf78a77917d44fd2300a41cd088aa38a08cd Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 11 Aug 2021 13:30:39 -0400 Subject: [PATCH 047/269] Send the warning to the parameterized stderr --- src/allmydata/scripts/runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/scripts/runner.py b/src/allmydata/scripts/runner.py index ba979fdfa..89265590c 100644 --- a/src/allmydata/scripts/runner.py +++ b/src/allmydata/scripts/runner.py @@ -239,7 +239,7 @@ def run(configFactory=Options, argv=sys.argv, stdout=sys.stdout, stderr=sys.stde :raise SystemExit: Always raised after the run is complete. """ if six.PY3: - print(PYTHON_3_WARNING, file=sys.stderr) + print(PYTHON_3_WARNING, file=stderr) if sys.platform == "win32": from allmydata.windows.fixups import initialize initialize() From 5f6ae1f8f5ccdd62eb427697e6404e0b3854b6e1 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 11 Aug 2021 13:30:52 -0400 Subject: [PATCH 048/269] Fix mis-merge: don't try to stripe argv[0] twice --- src/allmydata/scripts/runner.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/allmydata/scripts/runner.py b/src/allmydata/scripts/runner.py index 89265590c..d6c1bfdc7 100644 --- a/src/allmydata/scripts/runner.py +++ b/src/allmydata/scripts/runner.py @@ -325,7 +325,8 @@ def _run_with_reactor(reactor, config, argv, stdout, stderr): :param twisted.python.usage.Options config: The config object to use to parse the argument list. - :param argv: See ``run``. + :param [str] argv: The argument list to parse, *excluding* the name of the + program being run. :param stdout: See ``run``. :param stderr: See ``run``. @@ -334,7 +335,7 @@ def _run_with_reactor(reactor, config, argv, stdout, stderr): """ _setup_coverage(reactor, argv) - argv = list(map(argv_to_unicode, argv[1:])) + argv = list(map(argv_to_unicode, argv)) d = defer.maybeDeferred( parse_or_exit_with_explanation_with_config, config, From 2244f0374e8887fd258c98f63f8863891ccdea27 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 11 Aug 2021 13:31:07 -0400 Subject: [PATCH 049/269] fail more informatively --- src/allmydata/test/test_runner.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/allmydata/test/test_runner.py b/src/allmydata/test/test_runner.py index 48e3f4f21..4ec7a2153 100644 --- a/src/allmydata/test/test_runner.py +++ b/src/allmydata/test/test_runner.py @@ -338,7 +338,12 @@ class RunNode(common_util.SignalMixin, unittest.TestCase, pollmixin.PollMixin): u"--hostname", u"127.0.0.1", ]) - self.assertEqual(returncode, 0) + self.assertEqual( + returncode, + 0, + "stdout: {!r}\n" + "stderr: {!r}\n", + ) # This makes sure that node.url is written, which allows us to # detect when the introducer restarts in _node_has_restarted below. From 0176583e7537a2eae0fa3b4a2d29c2828638406c Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 11 Aug 2021 13:32:04 -0400 Subject: [PATCH 050/269] Get a text-mode FilePath in a py3 compatible way --- src/allmydata/test/cli/test_run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/cli/test_run.py b/src/allmydata/test/cli/test_run.py index c4342b3fe..28613e8c1 100644 --- a/src/allmydata/test/cli/test_run.py +++ b/src/allmydata/test/cli/test_run.py @@ -157,7 +157,7 @@ class RunTests(SyncTestCase): If the pidfile exists but does not contain a numeric value, a complaint to this effect is written to stderr. """ - basedir = FilePath(self.mktemp().decode("ascii")) + basedir = FilePath(self.mktemp()).asTextMode() basedir.makedirs() basedir.child(u"twistd.pid").setContent(b"foo") basedir.child(u"tahoe-client.tac").setContent(b"") From d56c218586b79393afe296b54bd35ce99a69a381 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 11 Aug 2021 14:25:24 -0400 Subject: [PATCH 051/269] Options are defined with unicode now; argv better be unicode. --- src/allmydata/scripts/runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/scripts/runner.py b/src/allmydata/scripts/runner.py index d6c1bfdc7..5c966b531 100644 --- a/src/allmydata/scripts/runner.py +++ b/src/allmydata/scripts/runner.py @@ -138,7 +138,7 @@ def parse_or_exit_with_explanation_with_config(config, argv, stdout, stderr): :param allmydata.scripts.runner.Options config: An instance of the argument-parsing class to use. - :param [str] argv: The argument list to parse, including the name of the + :param [unicode] argv: The argument list to parse, including the name of the program being run as ``argv[0]``. :param stdout: The file-like object to use as stdout. From 4d41e30ce9c3c2bc15f8e6af93a0afc49b002e6d Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 11 Aug 2021 14:25:40 -0400 Subject: [PATCH 052/269] Just pass unicode at this layer --- src/allmydata/test/test_runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/test_runner.py b/src/allmydata/test/test_runner.py index 4ec7a2153..c60dd4f01 100644 --- a/src/allmydata/test/test_runner.py +++ b/src/allmydata/test/test_runner.py @@ -99,7 +99,7 @@ class ParseOptionsTests(SyncTestCase): """ tricky = u"\u2621" try: - parse_options([unicode_to_argv(tricky)]) + parse_options([tricky]) except usage.error as e: self.assertEqual( b"Unknown command: " + tricky.encode("utf-8"), From 1d75bbfd72f3a4c0ad84946a6e7f1a07fa39a0e4 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 11 Aug 2021 14:25:47 -0400 Subject: [PATCH 053/269] `str` is a kind of weird future thing; coerce another way --- src/allmydata/test/test_runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/test_runner.py b/src/allmydata/test/test_runner.py index c60dd4f01..c9ad4d97c 100644 --- a/src/allmydata/test/test_runner.py +++ b/src/allmydata/test/test_runner.py @@ -103,7 +103,7 @@ class ParseOptionsTests(SyncTestCase): except usage.error as e: self.assertEqual( b"Unknown command: " + tricky.encode("utf-8"), - str(e) + b"{}".format(e), ) From fd3d3bc68823c78f36b14f3c610e2182051a9d91 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 11 Aug 2021 14:27:21 -0400 Subject: [PATCH 054/269] Give the py3 static checker something to resolve `unicode` to It's pretty much just a bug in the static checker. :/ --- src/allmydata/scripts/runner.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/allmydata/scripts/runner.py b/src/allmydata/scripts/runner.py index 5c966b531..93875a167 100644 --- a/src/allmydata/scripts/runner.py +++ b/src/allmydata/scripts/runner.py @@ -9,6 +9,7 @@ if PY2: import os, sys from six.moves import StringIO +from past.builtins import unicode import six try: From 975f268d8dd20505d075a4815eab7b2480471ed6 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 11 Aug 2021 14:39:32 -0400 Subject: [PATCH 055/269] Provide enough output to debug the failure --- src/allmydata/test/test_runner.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/allmydata/test/test_runner.py b/src/allmydata/test/test_runner.py index c9ad4d97c..f7fe538f1 100644 --- a/src/allmydata/test/test_runner.py +++ b/src/allmydata/test/test_runner.py @@ -144,7 +144,11 @@ class BinTahoe(common_util.SignalMixin, unittest.TestCase): tricky = u"\u00F6" out, err, returncode = run_bintahoe([tricky]) self.assertEqual(returncode, 1) - self.assertIn(u"Unknown command: " + tricky, out) + self.assertIn( + u"Unknown command: " + tricky, + out, + "stdout: {!r}\nstderr: {!r}".format(out, err), + ) def test_with_python_options(self): """ From 6931d10ace1a7415395918067ab3996f004e17f8 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 11 Aug 2021 14:40:22 -0400 Subject: [PATCH 056/269] Fix mis-merge: use argv parameter instead of sys.argv --- src/allmydata/scripts/runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/scripts/runner.py b/src/allmydata/scripts/runner.py index 93875a167..a818aafe8 100644 --- a/src/allmydata/scripts/runner.py +++ b/src/allmydata/scripts/runner.py @@ -170,7 +170,7 @@ def parse_or_exit_with_explanation_with_config(config, argv, stdout, stderr): # On Python 2 the string may turn into a unicode string, e.g. the error # may be unicode, in which case it will print funny. Once we're on # Python 3 we can just drop the ensure_str(). - print(six.ensure_str("%s: %s\n" % (sys.argv[0], e)), file=stdout) + print(six.ensure_str("%s: %s\n" % (argv[0], e)), file=stdout) sys.exit(1) return config From 13dae392cba3cef3d83213a0d15b7d8417f64763 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 11 Aug 2021 14:41:18 -0400 Subject: [PATCH 057/269] Go with a shorter name --- src/allmydata/scripts/runner.py | 4 ++-- src/allmydata/test/common_util.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/allmydata/scripts/runner.py b/src/allmydata/scripts/runner.py index a818aafe8..d9cd6e720 100644 --- a/src/allmydata/scripts/runner.py +++ b/src/allmydata/scripts/runner.py @@ -129,7 +129,7 @@ def parse_options(argv, config=None): raise return config -def parse_or_exit_with_explanation_with_config(config, argv, stdout, stderr): +def parse_or_exit(config, argv, stdout, stderr): """ Parse Tahoe-LAFS CLI arguments and return a configuration object if they are valid. @@ -338,7 +338,7 @@ def _run_with_reactor(reactor, config, argv, stdout, stderr): argv = list(map(argv_to_unicode, argv)) d = defer.maybeDeferred( - parse_or_exit_with_explanation_with_config, + parse_or_exit, config, argv, stdout, diff --git a/src/allmydata/test/common_util.py b/src/allmydata/test/common_util.py index 4db217705..b5229ca11 100644 --- a/src/allmydata/test/common_util.py +++ b/src/allmydata/test/common_util.py @@ -133,7 +133,7 @@ def run_cli_native(verb, *args, **kwargs): d = defer.succeed(argv) d.addCallback( partial( - runner.parse_or_exit_with_explanation_with_config, + runner.parse_or_exit, runner.Options(), ), stdout=stdout, From b56a95684310c0897c90ad2f7243c3a4d3e6bcb7 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 11 Aug 2021 15:42:21 -0400 Subject: [PATCH 058/269] Sort out this gross error reporting encoding/decoding mess A little, anyway --- src/allmydata/scripts/runner.py | 29 +++++++++++++++--- src/allmydata/test/test_runner.py | 50 +++++++++++++++++++++++++++++-- 2 files changed, 72 insertions(+), 7 deletions(-) diff --git a/src/allmydata/scripts/runner.py b/src/allmydata/scripts/runner.py index d9cd6e720..454c42c85 100644 --- a/src/allmydata/scripts/runner.py +++ b/src/allmydata/scripts/runner.py @@ -167,10 +167,31 @@ def parse_or_exit(config, argv, stdout, stderr): while hasattr(c, 'subOptions'): c = c.subOptions print(str(c), file=stdout) - # On Python 2 the string may turn into a unicode string, e.g. the error - # may be unicode, in which case it will print funny. Once we're on - # Python 3 we can just drop the ensure_str(). - print(six.ensure_str("%s: %s\n" % (argv[0], e)), file=stdout) + # On Python 2 the exception may hold non-ascii in a byte string. This + # makes it impossible to convert the exception to any kind of string + # using str() or unicode(). So, reach inside and get what we need. + # + # Then, since we are on Python 2, turn it into some entirely safe + # ascii that will survive being written to stdout without causing too + # much damage in the process. + # + # As a result, non-ascii will not be rendered correctly but instead as + # escape sequences. At least this can go away when we're done with + # Python 2 support. + if PY2: + exc_text = e.args[0].decode( + "utf-8", + ).encode( + "ascii", + errors="backslashreplace", + ).decode( + "ascii", + ) + else: + exc_text = unicode(e) + exc_bytes = six.ensure_binary(exc_text, "utf-8") + msg_bytes = b"%s: %s\n" % (six.ensure_binary(argv[0]), exc_bytes) + print(six.ensure_text(msg_bytes, "utf-8"), file=stdout) sys.exit(1) return config diff --git a/src/allmydata/test/test_runner.py b/src/allmydata/test/test_runner.py index f7fe538f1..a420581e9 100644 --- a/src/allmydata/test/test_runner.py +++ b/src/allmydata/test/test_runner.py @@ -24,6 +24,16 @@ import six from testtools import ( skipUnless, ) +from testtools.matchers import ( + MatchesListwise, + MatchesAny, + Contains, + Equals, + Always, +) +from testtools.twistedsupport import ( + succeeded, +) from eliot import ( log_call, ) @@ -55,6 +65,7 @@ from .common import ( from .common_util import ( parse_cli, run_cli, + run_cli_unicode, ) from .cli_node_api import ( CLINodeAPI, @@ -97,7 +108,7 @@ class ParseOptionsTests(SyncTestCase): does not exist and which also contains non-ascii characters, the exception it raises includes the subcommand encoded as UTF-8. """ - tricky = u"\u2621" + tricky = u"\u00F6" try: parse_options([tricky]) except usage.error as e: @@ -107,6 +118,35 @@ class ParseOptionsTests(SyncTestCase): ) +class ParseOrExitTests(SyncTestCase): + """ + Tests for ``parse_or_exit``. + """ + def test_nonascii_error_content(self): + """ + ``parse_or_exit`` can report errors that include non-ascii content. + """ + tricky = u"\u00F6" + self.assertThat( + run_cli_unicode(tricky, [], encoding="utf-8"), + succeeded( + MatchesListwise([ + # returncode + Equals(1), + # stdout + MatchesAny( + # Python 2 + Contains(u"Unknown command: \\xf6"), + # Python 3 + Contains(u"Unknown command: \xf6"), + ), + # stderr, + Always() + ]), + ), + ) + + @log_call(action_type="run-bin-tahoe") def run_bintahoe(extra_argv, python_options=None): """ @@ -143,11 +183,15 @@ class BinTahoe(common_util.SignalMixin, unittest.TestCase): """ tricky = u"\u00F6" out, err, returncode = run_bintahoe([tricky]) + if PY2: + expected = u"Unknown command: \\xf6" + else: + expected = u"Unknown command: \xf6" self.assertEqual(returncode, 1) self.assertIn( - u"Unknown command: " + tricky, + expected, out, - "stdout: {!r}\nstderr: {!r}".format(out, err), + "expected {!r} not found in {!r}\nstderr: {!r}".format(expected, out, err), ) def test_with_python_options(self): From 893d21fcbbbe330434a8ad96e3076e1746bb381c Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 11 Aug 2021 16:46:29 -0400 Subject: [PATCH 059/269] Fix the UsageError closer in the Py2 codepath we already have for it --- src/allmydata/scripts/runner.py | 43 +++++++++++++------------------ src/allmydata/test/test_runner.py | 2 +- 2 files changed, 19 insertions(+), 26 deletions(-) diff --git a/src/allmydata/scripts/runner.py b/src/allmydata/scripts/runner.py index 454c42c85..185af2322 100644 --- a/src/allmydata/scripts/runner.py +++ b/src/allmydata/scripts/runner.py @@ -120,9 +120,23 @@ def parse_options(argv, config=None): config.parseOptions(argv) except usage.error as e: if six.PY2: - # Exceptions must stringify to bytes on Python 2. + # On Python 2 the exception may hold non-ascii in a byte string. + # This makes it impossible to convert the exception to any kind of + # string using str() or unicode(). It could also hold non-ascii + # in a unicode string which still makes it difficult to convert it + # to a byte string later. + # + # So, reach inside and turn it into some entirely safe ascii byte + # strings that will survive being written to stdout without + # causing too much damage in the process. + # + # As a result, non-ascii will not be rendered correctly but + # instead as escape sequences. At least this can go away when + # we're done with Python 2 support. raise usage.error(*( - arg.encode("utf-8") if isinstance(arg, unicode) else arg + arg.encode("ascii", errors="backslashreplace") + if isinstance(arg, unicode) + else arg.decode("utf-8").encode("ascii", errors="backslashreplace") for arg in e.args )) @@ -167,29 +181,8 @@ def parse_or_exit(config, argv, stdout, stderr): while hasattr(c, 'subOptions'): c = c.subOptions print(str(c), file=stdout) - # On Python 2 the exception may hold non-ascii in a byte string. This - # makes it impossible to convert the exception to any kind of string - # using str() or unicode(). So, reach inside and get what we need. - # - # Then, since we are on Python 2, turn it into some entirely safe - # ascii that will survive being written to stdout without causing too - # much damage in the process. - # - # As a result, non-ascii will not be rendered correctly but instead as - # escape sequences. At least this can go away when we're done with - # Python 2 support. - if PY2: - exc_text = e.args[0].decode( - "utf-8", - ).encode( - "ascii", - errors="backslashreplace", - ).decode( - "ascii", - ) - else: - exc_text = unicode(e) - exc_bytes = six.ensure_binary(exc_text, "utf-8") + exc_str = str(e) + exc_bytes = six.ensure_binary(exc_str, "utf-8") msg_bytes = b"%s: %s\n" % (six.ensure_binary(argv[0]), exc_bytes) print(six.ensure_text(msg_bytes, "utf-8"), file=stdout) sys.exit(1) diff --git a/src/allmydata/test/test_runner.py b/src/allmydata/test/test_runner.py index a420581e9..44c7e1bee 100644 --- a/src/allmydata/test/test_runner.py +++ b/src/allmydata/test/test_runner.py @@ -113,7 +113,7 @@ class ParseOptionsTests(SyncTestCase): parse_options([tricky]) except usage.error as e: self.assertEqual( - b"Unknown command: " + tricky.encode("utf-8"), + b"Unknown command: \\xf6", b"{}".format(e), ) From 85ba6567ba8798d78e6950283d7a53d8c2c30c1e Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 12 Aug 2021 12:11:39 -0400 Subject: [PATCH 060/269] Try to make sure fixed argv is used on Py27+Windows Previous version that rebound sys.argv didn't work so well with early binding used by some some functions for default argument values. --- src/allmydata/windows/fixups.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/allmydata/windows/fixups.py b/src/allmydata/windows/fixups.py index e8d05a659..53eb14d53 100644 --- a/src/allmydata/windows/fixups.py +++ b/src/allmydata/windows/fixups.py @@ -188,7 +188,17 @@ def initialize(): # for example, the Python interpreter or any options passed to it, or runner # scripts such as 'coverage run'. It works even if there are no such arguments, # as in the case of a frozen executable created by bb-freeze or similar. - sys.argv = argv[-len(sys.argv):] + # + # Also, modify sys.argv in place. If any code has already taken a + # reference to the original argument list object then this ensures that + # code sees the new values. This reliance on mutation of shared state is, + # of course, awful. Why does this function even modify sys.argv? Why not + # have a function that *returns* the properly initialized argv as a new + # list? I don't know. + # + # At least Python 3 gets sys.argv correct so before very much longer we + # should be able to fix this bad design by deleting it. + sys.argv[:] = argv[-len(sys.argv):] def a_console(handle): From acc8cbd28b1e1bf44d529c5dc7d80500fcd71e23 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 12 Aug 2021 16:50:01 -0400 Subject: [PATCH 061/269] Remove use of the mock module from test_status --- src/allmydata/scripts/tahoe_status.py | 72 ++++++++++++-------- src/allmydata/test/cli/test_status.py | 98 ++++++++++++--------------- 2 files changed, 86 insertions(+), 84 deletions(-) diff --git a/src/allmydata/scripts/tahoe_status.py b/src/allmydata/scripts/tahoe_status.py index 75bc775f8..5d61fcf5b 100644 --- a/src/allmydata/scripts/tahoe_status.py +++ b/src/allmydata/scripts/tahoe_status.py @@ -17,15 +17,17 @@ import json from .common import BaseOptions from allmydata.scripts.common import get_default_nodedir -from allmydata.scripts.common_http import do_http, BadResponse +from allmydata.scripts.common_http import BadResponse from allmydata.util.abbreviate import abbreviate_space, abbreviate_time from allmydata.util.encodingutil import argv_to_abspath -def _get_json_for_fragment(options, fragment, method='GET', post_args=None): +def _get_request_parameters_for_fragment(options, fragment, method, post_args): """ - returns the JSON for a particular URI-fragment (to which is - pre-pended the node's URL) + Get parameters for ``do_http`` for requesting the given fragment. + + :return dict: A dictionary suitable for use as keyword arguments to + ``do_http``. """ nodeurl = options['node-url'] if nodeurl.endswith('/'): @@ -40,7 +42,17 @@ def _get_json_for_fragment(options, fragment, method='GET', post_args=None): body = '' if post_args is not None: raise ValueError("post_args= only valid for POST method") - resp = do_http(method, url, body=body.encode("utf-8")) + return dict( + method=method, + url=url, + body=body.encode("utf-8"), + ) + + +def _handle_response_for_fragment(resp, nodeurl): + """ + Inspect an HTTP response and return the parsed payload, if possible. + """ if isinstance(resp, BadResponse): # specifically NOT using format_http_error() here because the # URL is pretty sensitive (we're doing /uri/). @@ -55,12 +67,6 @@ def _get_json_for_fragment(options, fragment, method='GET', post_args=None): return parsed -def _get_json_for_cap(options, cap): - return _get_json_for_fragment( - options, - 'uri/%s?t=json' % url_quote(cap), - ) - def pretty_progress(percent, size=10, output_ascii=False): """ Displays a unicode or ascii based progress bar of a certain @@ -251,7 +257,10 @@ def render_recent(verbose, stdout, status_data): print(u" Skipped {} non-upload/download operations; use --verbose to see".format(skipped), file=stdout) -def do_status(options): +def do_status(options, do_http=None): + if do_http is None: + from allmydata.scripts.common_http import do_http + nodedir = options["node-directory"] with open(os.path.join(nodedir, u'private', u'api_auth_token'), 'r') as f: token = f.read().strip() @@ -260,25 +269,30 @@ def do_status(options): # do *all* our data-retrievals first in case there's an error try: - status_data = _get_json_for_fragment( - options, - 'status?t=json', - method='POST', - post_args=dict( - t='json', - token=token, - ) + status_data = _handle_response_for_fragment( + do_http(**_get_request_parameters_for_fragment( + options, + 'status?t=json', + method='POST', + post_args=dict( + t='json', + token=token, + ), + )), + options['node-url'], ) - statistics_data = _get_json_for_fragment( - options, - 'statistics?t=json', - method='POST', - post_args=dict( - t='json', - token=token, - ) + statistics_data = _handle_response_for_fragment( + do_http(**_get_request_parameters_for_fragment( + options, + 'statistics?t=json', + method='POST', + post_args=dict( + t='json', + token=token, + ), + )), + options['node-url'], ) - except Exception as e: print(u"failed to retrieve data: %s" % str(e), file=options.stderr) return 2 diff --git a/src/allmydata/test/cli/test_status.py b/src/allmydata/test/cli/test_status.py index 724661211..0d9e23c83 100644 --- a/src/allmydata/test/cli/test_status.py +++ b/src/allmydata/test/cli/test_status.py @@ -12,7 +12,6 @@ if PY2: from six import ensure_text import os -import mock import tempfile from io import BytesIO, StringIO from os.path import join @@ -22,8 +21,8 @@ from twisted.internet import defer from allmydata.mutable.publish import MutableData from allmydata.scripts.common_http import BadResponse -from allmydata.scripts.tahoe_status import _get_json_for_fragment -from allmydata.scripts.tahoe_status import _get_json_for_cap +from allmydata.scripts.tahoe_status import _handle_response_for_fragment +from allmydata.scripts.tahoe_status import _get_request_parameters_for_fragment from allmydata.scripts.tahoe_status import pretty_progress from allmydata.scripts.tahoe_status import do_status from allmydata.web.status import marshal_json @@ -148,9 +147,7 @@ class CommandStatus(unittest.TestCase): def setUp(self): self.options = _FakeOptions() - @mock.patch('allmydata.scripts.tahoe_status.do_http') - @mock.patch('sys.stdout', StringIO()) - def test_no_operations(self, http): + def test_no_operations(self): values = [ StringIO(ensure_text(json.dumps({ "active": [], @@ -165,12 +162,11 @@ class CommandStatus(unittest.TestCase): } }))), ] - http.side_effect = lambda *args, **kw: values.pop(0) - do_status(self.options) + def do_http(*args, **kw): + return values.pop(0) + do_status(self.options, do_http) - @mock.patch('allmydata.scripts.tahoe_status.do_http') - @mock.patch('sys.stdout', StringIO()) - def test_simple(self, http): + def test_simple(self): recent_items = active_items = [ UploadStatus(), DownloadStatus(b"abcd", 12345), @@ -201,80 +197,72 @@ class CommandStatus(unittest.TestCase): } }).encode("utf-8")), ] - http.side_effect = lambda *args, **kw: values.pop(0) - do_status(self.options) + def do_http(*args, **kw): + return values.pop(0) + do_status(self.options, do_http) - @mock.patch('allmydata.scripts.tahoe_status.do_http') - def test_fetch_error(self, http): - - def boom(*args, **kw): + def test_fetch_error(self): + def do_http(*args, **kw): raise RuntimeError("boom") - http.side_effect = boom - do_status(self.options) + do_status(self.options, do_http) class JsonHelpers(unittest.TestCase): - @mock.patch('allmydata.scripts.tahoe_status.do_http') - def test_bad_response(self, http): - http.return_value = BadResponse('the url', 'some err') + def test_bad_response(self): + def do_http(*args, **kw): + return with self.assertRaises(RuntimeError) as ctx: - _get_json_for_fragment({'node-url': 'http://localhost:1234'}, '/fragment') - self.assertTrue( - "Failed to get" in str(ctx.exception) + _handle_response_for_fragment( + BadResponse('the url', 'some err'), + 'http://localhost:1234', + ) + self.assertIn( + "Failed to get", + str(ctx.exception), ) - @mock.patch('allmydata.scripts.tahoe_status.do_http') - def test_happy_path(self, http): - http.return_value = StringIO('{"some": "json"}') - resp = _get_json_for_fragment({'node-url': 'http://localhost:1234/'}, '/fragment/') - self.assertEqual(resp, dict(some='json')) - - @mock.patch('allmydata.scripts.tahoe_status.do_http') - def test_happy_path_post(self, http): - http.return_value = StringIO('{"some": "json"}') - resp = _get_json_for_fragment( - {'node-url': 'http://localhost:1234/'}, - '/fragment/', - method='POST', - post_args={'foo': 'bar'} + def test_happy_path(self): + resp = _handle_response_for_fragment( + StringIO('{"some": "json"}'), + 'http://localhost:1234/', ) self.assertEqual(resp, dict(some='json')) - @mock.patch('allmydata.scripts.tahoe_status.do_http') - def test_happy_path_for_cap(self, http): - http.return_value = StringIO('{"some": "json"}') - resp = _get_json_for_cap({'node-url': 'http://localhost:1234'}, 'fake cap') + def test_happy_path_post(self): + resp = _handle_response_for_fragment( + StringIO('{"some": "json"}'), + 'http://localhost:1234/', + ) self.assertEqual(resp, dict(some='json')) - @mock.patch('allmydata.scripts.tahoe_status.do_http') - def test_no_data_returned(self, http): - http.return_value = StringIO('null') - + def test_no_data_returned(self): with self.assertRaises(RuntimeError) as ctx: - _get_json_for_cap({'node-url': 'http://localhost:1234'}, 'fake cap') - self.assertTrue('No data from' in str(ctx.exception)) + _handle_response_for_fragment(StringIO('null'), 'http://localhost:1234') + self.assertIn('No data from', str(ctx.exception)) def test_no_post_args(self): with self.assertRaises(ValueError) as ctx: - _get_json_for_fragment( + _get_request_parameters_for_fragment( {'node-url': 'http://localhost:1234'}, '/fragment', method='POST', post_args=None, ) - self.assertTrue( - "Must pass post_args" in str(ctx.exception) + self.assertIn( + "Must pass post_args", + str(ctx.exception), ) def test_post_args_for_get(self): with self.assertRaises(ValueError) as ctx: - _get_json_for_fragment( + _get_request_parameters_for_fragment( {'node-url': 'http://localhost:1234'}, '/fragment', method='GET', post_args={'foo': 'bar'} ) - self.assertTrue( - "only valid for POST" in str(ctx.exception) + self.assertIn( + "only valid for POST", + str(ctx.exception), ) From 52a61e8f63f36080597956d4b9543eee035c9a9f Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 12 Aug 2021 16:50:18 -0400 Subject: [PATCH 062/269] news fragment --- newsfragments/3525.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3525.minor diff --git a/newsfragments/3525.minor b/newsfragments/3525.minor new file mode 100644 index 000000000..e69de29bb From 9a8faae28319c40e7e480a451c10072d90ca822b Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 13 Aug 2021 09:24:00 -0400 Subject: [PATCH 063/269] Remove PYTHONIOENCODING, set for towncrier, no longer run here --- tox.ini | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/tox.ini b/tox.ini index 13b2b999c..9b0f71038 100644 --- a/tox.ini +++ b/tox.ini @@ -114,13 +114,6 @@ commands = [testenv:codechecks] basepython = python2.7 setenv = - # Workaround an error when towncrier is run under the VCS hook, - # https://stackoverflow.com/a/4027726/624787: - # File "/home/rpatterson/src/work/sfu/tahoe-lafs/.tox/codechecks/lib/python2.7/site-packages/towncrier/check.py", line 44, in __main - # .decode(getattr(sys.stdout, "encoding", "utf8")) - # `TypeError: decode() argument 1 must be string, not None` - PYTHONIOENCODING=utf_8 - # If no positional arguments are given, try to run the checks on the # entire codebase, including various pieces of supporting code. DEFAULT_FILES=src integration static misc setup.py @@ -190,7 +183,7 @@ passenv = TAHOE_LAFS_* PIP_* SUBUNITREPORTER_* USERPROFILE HOMEDRIVE HOMEPATH HO whitelist_externals = git deps = - # see comment in [testenv] about "certifi" + # see comment in [testenv] about "certifi" certifi towncrier==21.3.0 commands = From a591c2c36d6f05535a21848bd89c1c7047191e23 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 17 Aug 2021 10:23:37 -0400 Subject: [PATCH 064/269] news fragment --- newsfragments/3764.documentation | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/3764.documentation diff --git a/newsfragments/3764.documentation b/newsfragments/3764.documentation new file mode 100644 index 000000000..d473cd27c --- /dev/null +++ b/newsfragments/3764.documentation @@ -0,0 +1 @@ +The Great Black Swamp proposed specification now includes sample interactions to demonstrate expected usage patterns. \ No newline at end of file From 23212fc816cf5f75e1238ae1ccb9827c64fd6d67 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 17 Aug 2021 14:01:44 -0400 Subject: [PATCH 065/269] add some example client/server interactions --- docs/proposed/http-storage-node-protocol.rst | 130 +++++++++++++++++++ 1 file changed, 130 insertions(+) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index ad9dd30bc..623cd4f12 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -576,6 +576,136 @@ Just like ``GET /v1/mutable/:storage_index``. Advise the server the data read from the indicated share was corrupt. Just like the immutable version. +Sample Interactions +------------------- + +Immutable Data +~~~~~~~~~~~~~~ + +1. Create a bucket for storage index ``AAAAAAAAAAAAAAAA`` to hold two immutable shares, discovering that share ``1`` was already uploaded:: + + PUT /v1/immutable/AAAAAAAAAAAAAAAA + {"renew-secret": "efgh", "cancel-secret": "ijkl", + "share-numbers": [1, 7], "allocated-size": 48} + + 200 OK + {"already-have": [1], "allocated": [7]} + +#. Upload the content for immutable share ``7``:: + + PUT /v1/immutable/AAAAAAAAAAAAAAAA/7 + Content-Range: bytes 0-15/48 + + + 200 OK + + PUT /v1/immutable/AAAAAAAAAAAAAAAA/7 + Content-Range: bytes 16-31/48 + + + 200 OK + + PUT /v1/immutable/AAAAAAAAAAAAAAAA/7 + Content-Range: bytes 32-47/48 + + + 201 CREATED + +#. Download the content of the previously uploaded immutable share ``7``:: + + GET /v1/immutable/AAAAAAAAAAAAAAAA?share=7&offset=0&size=48 + + 200 OK + + +#. Renew the lease on all immutable shares in bucket ``AAAAAAAAAAAAAAAA``:: + + POST /v1/lease/AAAAAAAAAAAAAAAA + {"renew-secret": "efgh"} + + 204 NO CONTENT + +Mutable Data +~~~~~~~~~~~~ + +1. Create mutable share number ``3`` with ``10`` bytes of data in slot ``BBBBBBBBBBBBBBBB``:: + + POST /v1/mutable/BBBBBBBBBBBBBBBB/read-test-write + { + "secrets": { + "write-enabler": "abcd", + "lease-renew": "efgh", + "lease-cancel": "ijkl" + }, + "test-write-vectors": { + 3: { + "test": [{ + "offset": 0, + "size": 1, + "operator": "eq", + "specimen": "" + }], + "write": [{ + "offset": 0, + "data": "xxxxxxxxxx" + }], + "new-length": 10 + } + }, + "read-vector": [] + } + + 200 OK + { + "success": true, + "data": [] + } + +#. Safely rewrite the contents of a known version of mutble share number ``3`` (or fail):: + + POST /v1/mutable/BBBBBBBBBBBBBBBB/read-test-write + { + "secrets": { + "write-enabler": "abcd", + "lease-renew": "efgh", + "lease-cancel": "ijkl" + }, + "test-write-vectors": { + 3: { + "test": [{ + "offset": 0, + "size": , + "operator": "eq", + "specimen": "" + }], + "write": [{ + "offset": 0, + "data": "yyyyyyyyyy" + }], + "new-length": 10 + } + }, + "read-vector": [] + } + + 200 OK + { + "success": true, + "data": [] + } + +#. Download the contents of share number ``3``:: + + GET /v1/mutable/BBBBBBBBBBBBBBBB?share=3&offset=0&size=10 + + +#. Renew the lease on previously uploaded mutable share in slot ``BBBBBBBBBBBBBBBB``:: + + POST /v1/lease/BBBBBBBBBBBBBBBB + {"renew-secret": "efgh"} + + 204 NO CONTENT + .. _RFC 7469: https://tools.ietf.org/html/rfc7469#section-2.4 .. _RFC 7049: https://tools.ietf.org/html/rfc7049#section-4 From 65f71135e19efa7d56ed927d4bbd75f047898f7e Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 17 Aug 2021 14:10:50 -0400 Subject: [PATCH 066/269] Spell the endpoint correctly --- docs/proposed/http-storage-node-protocol.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index 623cd4f12..3592a1532 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -584,7 +584,7 @@ Immutable Data 1. Create a bucket for storage index ``AAAAAAAAAAAAAAAA`` to hold two immutable shares, discovering that share ``1`` was already uploaded:: - PUT /v1/immutable/AAAAAAAAAAAAAAAA + POST /v1/immutable/AAAAAAAAAAAAAAAA {"renew-secret": "efgh", "cancel-secret": "ijkl", "share-numbers": [1, 7], "allocated-size": 48} From 47dfe5d4f0c75e49b3561a92c019d7063bd87d00 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 17 Aug 2021 14:11:08 -0400 Subject: [PATCH 067/269] fix typo --- docs/proposed/http-storage-node-protocol.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index 3592a1532..1f7df84c0 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -661,7 +661,7 @@ Mutable Data "data": [] } -#. Safely rewrite the contents of a known version of mutble share number ``3`` (or fail):: +#. Safely rewrite the contents of a known version of mutable share number ``3`` (or fail):: POST /v1/mutable/BBBBBBBBBBBBBBBB/read-test-write { From 49300fc0808307913216d9a522ff83601182e236 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 17 Aug 2021 14:58:30 -0400 Subject: [PATCH 068/269] news fragment --- newsfragments/3765.documentation | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/3765.documentation diff --git a/newsfragments/3765.documentation b/newsfragments/3765.documentation new file mode 100644 index 000000000..a3b59c4d6 --- /dev/null +++ b/newsfragments/3765.documentation @@ -0,0 +1 @@ +The Great Black Swamp proposed specification now includes a glossary. \ No newline at end of file From 0f78d8df2561dd2509cec991abac78d65e16069d Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 17 Aug 2021 14:58:34 -0400 Subject: [PATCH 069/269] a glossary --- docs/proposed/http-storage-node-protocol.rst | 41 ++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index ad9dd30bc..cc2d15627 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -13,6 +13,47 @@ Specifically, it should be possible to implement a Tahoe-LAFS storage server wit The Tahoe-LAFS client will also need to change but it is not expected that it will be noticably simplified by this change (though this may be the first step towards simplifying it). +Glossary +-------- + +.. glossary:: + + `Foolscap `_ + an RPC/RMI (Remote Procedure Call / Remote Method Invocation) protocol for use with Twisted + + storage server + a Tahoe-LAFS process configured to offer storage and reachable over the network for store and retrieve operations + + introducer + a Tahoe-LAFS process at a known location configured to re-publish announcements about the location of storage servers + + fURL + a self-authenticating URL-like string which can be used to locate a remote object using the Foolscap protocol + + lease + state associated with a share informing a storage server of the duration of storage desired by a client + + share + a single unit of client-provided arbitrary data to be stored by a storage server + (in practice, one of the outputs of applying ZFEC encoding to some ciphertext with some additional metadata attached) + + bucket + a group of one or more immutable shares held by a storage server and having a common storage index + + slot + a group of one or more mutable shares held by a storage server and having a common storage index + (sometimes "slot" is considered a synonym for "storage index of a slot") + + storage index + a short string which can address a slot or a bucket + (in practice, derived by hashing the encryption key associated with contents of that slot or bucket) + + write enabler + a short secret string which storage servers require to be presented before allowing mutation of any mutable share + + lease renew secret + a short secret string which storage servers required to be presented before allowing a particular lease to be renewed + Motivation ---------- From cad7604ba30a6338a8a6385db64bef8df9e93418 Mon Sep 17 00:00:00 2001 From: YashNRam13 Date: Wed, 18 Aug 2021 14:27:03 +0530 Subject: [PATCH 070/269] docs: Update README.rst --- README.rst | 9 +++------ newsfragments/3749.documentation | 1 + 2 files changed, 4 insertions(+), 6 deletions(-) create mode 100644 newsfragments/3749.documentation diff --git a/README.rst b/README.rst index 2cc6e38eb..99126c009 100644 --- a/README.rst +++ b/README.rst @@ -1,5 +1,5 @@ ====================================== -Free and Open decentralized data store +Free and Open Decentralized Data Store ====================================== |image0| @@ -48,13 +48,10 @@ Please read more about Tahoe-LAFS architecture `here `__. ✅ Installation --------------- -For more detailed instructions, read `docs/INSTALL.rst `__ . +For more detailed instructions, read `Installing Tahoe-LAFS `__. -- `Building Tahoe-LAFS on Windows `__ -- `OS-X Packaging `__ - -Once tahoe --version works, see `docs/running.rst `__ to learn how to set up your first Tahoe-LAFS node. +Once ``tahoe --version`` works, see `How to Run Tahoe-LAFS `__ to learn how to set up your first Tahoe-LAFS node. 🤖 Issues diff --git a/newsfragments/3749.documentation b/newsfragments/3749.documentation new file mode 100644 index 000000000..82e1e7856 --- /dev/null +++ b/newsfragments/3749.documentation @@ -0,0 +1 @@ +Fixes links to the installation and about Tahoe pages in the docs. From c5a03f4116bb3d83ed7e6213b784e8cc3a0a1528 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 18 Aug 2021 10:17:48 -0400 Subject: [PATCH 071/269] elaborate on ``already-have`` and ``allocated`` somewhat --- docs/proposed/http-storage-node-protocol.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index 1f7df84c0..dd084c397 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -422,6 +422,15 @@ However, we decided this does not matter because: therefore no proxy servers can perform any extra logging. * Tahoe-LAFS itself does not currently log HTTP request URLs. +The response includes ``already-have`` and ``allocated`` for two reasons: + +* If an upload is interrupted and the client loses its local state that lets it know it already uploaded some shares + then this allows it to discover this fact (by inspecting ``already-have``) and only upload the missing shares (indicated by ``allocated``). + +* If an upload has completed a client may still choose to re-balance storage by moving shares between servers. + This might be because a server has become unavailable and a remaining server needs to store more shares for the upload. + It could also just be that the client's preferred servers have changed. + ``PUT /v1/immutable/:storage_index/:share_number`` !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! From 602d4c5a9129e6312c1ed64d72dc4fb365f53ec4 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 19 Aug 2021 10:26:45 -0400 Subject: [PATCH 072/269] improve the "create lease" endpoint * Simplify some language using terms from our new glossary * explicitly state the two success-case behaviors * make the error-case behavior different from the success-case behavior * link to some tickets about future work in this area --- docs/proposed/http-storage-node-protocol.rst | 24 ++++++++++++-------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index 41a0a0fea..de0918b58 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -369,19 +369,26 @@ For example:: ``PUT /v1/lease/:storage_index`` !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -Create a new lease that applies to all shares for the given storage index. +Create a new lease on the bucket addressed by ``storage_index``. The details of the lease are encoded in the request body. For example:: {"renew-secret": "abcd", "cancel-secret": "efgh"} -If there are no shares for the given ``storage_index`` -then do nothing and return ``NO CONTENT``. - If the ``renew-secret`` value matches an existing lease -then that lease will be renewed instead. +then the expiration time of that lease will be changed to 31 days after the time of this operation. +If it does not match an existing lease +then a new lease will be created with this ``renew-secret`` which expires 31 days after the time of this operation. + +In these cases the response is ``NO CONTENT`` with an empty body. + +It is possible that the storage server will have no shares for the given ``storage_index`` because: + +* no such shares have ever been uploaded. +* a previous lease expired and the storage server reclaimed the storage by deleting the shares. + +In these cases the server takes no action and returns ``NOT FOUND``. -The lease expires after 31 days. Discussion `````````` @@ -391,10 +398,9 @@ We chose to put these values into the request body to make the URL simpler. Several behaviors here are blindly copied from the Foolscap-based storage server protocol. -* There is a cancel secret but there is no API to use it to cancel a lease. +* There is a cancel secret but there is no API to use it to cancel a lease (see ticket:3768). * The lease period is hard-coded at 31 days. -* There is no way to differentiate between success and an unknown **storage index**. -* There are separate **add** and **renew** lease APIs. +* There are separate **add** and **renew** lease APIs (see ticket:3773). These are not necessarily ideal behaviors but they are adopted to avoid any *semantic* changes between the Foolscap- and HTTP-based protocols. From 03c633cda0f92daa22c0dae36c900681a2ed27b6 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 19 Aug 2021 10:28:17 -0400 Subject: [PATCH 073/269] news fragment --- newsfragments/3763.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3763.minor diff --git a/newsfragments/3763.minor b/newsfragments/3763.minor new file mode 100644 index 000000000..e69de29bb From dfff187ad0ce631e175c1eb1a4e0b4ad16290d5f Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 19 Aug 2021 13:23:01 -0400 Subject: [PATCH 074/269] Make time pluggable to support better testing. --- src/allmydata/storage/server.py | 38 +++++++++++++++++---------------- 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py index 6cf6f6672..fb0cf1201 100644 --- a/src/allmydata/storage/server.py +++ b/src/allmydata/storage/server.py @@ -62,7 +62,8 @@ class StorageServer(service.MultiService, Referenceable): expiration_mode="age", expiration_override_lease_duration=None, expiration_cutoff_date=None, - expiration_sharetypes=("mutable", "immutable")): + expiration_sharetypes=("mutable", "immutable"), + get_current_time=time.time): service.MultiService.__init__(self) assert isinstance(nodeid, bytes) assert len(nodeid) == 20 @@ -114,6 +115,7 @@ class StorageServer(service.MultiService, Referenceable): expiration_cutoff_date, expiration_sharetypes) self.lease_checker.setServiceParent(self) + self._get_current_time = get_current_time def __repr__(self): return "" % (idlib.shortnodeid_b2a(self.my_nodeid),) @@ -264,7 +266,7 @@ class StorageServer(service.MultiService, Referenceable): # owner_num is not for clients to set, but rather it should be # curried into the PersonalStorageServer instance that is dedicated # to a particular owner. - start = time.time() + start = self._get_current_time() self.count("allocate") alreadygot = set() bucketwriters = {} # k: shnum, v: BucketWriter @@ -277,7 +279,7 @@ class StorageServer(service.MultiService, Referenceable): # goes into the share files themselves. It could also be put into a # separate database. Note that the lease should not be added until # the BucketWriter has been closed. - expire_time = time.time() + 31*24*60*60 + expire_time = self._get_current_time() + 31*24*60*60 lease_info = LeaseInfo(owner_num, renew_secret, cancel_secret, expire_time, self.my_nodeid) @@ -331,7 +333,7 @@ class StorageServer(service.MultiService, Referenceable): if bucketwriters: fileutil.make_dirs(os.path.join(self.sharedir, si_dir)) - self.add_latency("allocate", time.time() - start) + self.add_latency("allocate", self._get_current_time() - start) return alreadygot, bucketwriters def _iter_share_files(self, storage_index): @@ -351,26 +353,26 @@ class StorageServer(service.MultiService, Referenceable): def remote_add_lease(self, storage_index, renew_secret, cancel_secret, owner_num=1): - start = time.time() + start = self._get_current_time() self.count("add-lease") - new_expire_time = time.time() + 31*24*60*60 + new_expire_time = self._get_current_time() + 31*24*60*60 lease_info = LeaseInfo(owner_num, renew_secret, cancel_secret, new_expire_time, self.my_nodeid) for sf in self._iter_share_files(storage_index): sf.add_or_renew_lease(lease_info) - self.add_latency("add-lease", time.time() - start) + self.add_latency("add-lease", self._get_current_time() - start) return None def remote_renew_lease(self, storage_index, renew_secret): - start = time.time() + start = self._get_current_time() self.count("renew") - new_expire_time = time.time() + 31*24*60*60 + new_expire_time = self._get_current_time() + 31*24*60*60 found_buckets = False for sf in self._iter_share_files(storage_index): found_buckets = True sf.renew_lease(renew_secret, new_expire_time) - self.add_latency("renew", time.time() - start) + self.add_latency("renew", self._get_current_time() - start) if not found_buckets: raise IndexError("no such lease to renew") @@ -394,7 +396,7 @@ class StorageServer(service.MultiService, Referenceable): pass def remote_get_buckets(self, storage_index): - start = time.time() + start = self._get_current_time() self.count("get") si_s = si_b2a(storage_index) log.msg("storage: get_buckets %r" % si_s) @@ -402,7 +404,7 @@ class StorageServer(service.MultiService, Referenceable): for shnum, filename in self._get_bucket_shares(storage_index): bucketreaders[shnum] = BucketReader(self, filename, storage_index, shnum) - self.add_latency("get", time.time() - start) + self.add_latency("get", self._get_current_time() - start) return bucketreaders def get_leases(self, storage_index): @@ -563,7 +565,7 @@ class StorageServer(service.MultiService, Referenceable): :return LeaseInfo: Information for a new lease for a share. """ ownerid = 1 # TODO - expire_time = time.time() + 31*24*60*60 # one month + expire_time = self._get_current_time() + 31*24*60*60 # one month lease_info = LeaseInfo(ownerid, renew_secret, cancel_secret, expire_time, self.my_nodeid) @@ -599,7 +601,7 @@ class StorageServer(service.MultiService, Referenceable): See ``allmydata.interfaces.RIStorageServer`` for details about other parameters and return value. """ - start = time.time() + start = self._get_current_time() self.count("writev") si_s = si_b2a(storage_index) log.msg("storage: slot_writev %r" % si_s) @@ -640,7 +642,7 @@ class StorageServer(service.MultiService, Referenceable): self._add_or_renew_leases(remaining_shares, lease_info) # all done - self.add_latency("writev", time.time() - start) + self.add_latency("writev", self._get_current_time() - start) return (testv_is_good, read_data) def remote_slot_testv_and_readv_and_writev(self, storage_index, @@ -666,7 +668,7 @@ class StorageServer(service.MultiService, Referenceable): return share def remote_slot_readv(self, storage_index, shares, readv): - start = time.time() + start = self._get_current_time() self.count("readv") si_s = si_b2a(storage_index) lp = log.msg("storage: slot_readv %r %r" % (si_s, shares), @@ -675,7 +677,7 @@ class StorageServer(service.MultiService, Referenceable): # shares exist if there is a file for them bucketdir = os.path.join(self.sharedir, si_dir) if not os.path.isdir(bucketdir): - self.add_latency("readv", time.time() - start) + self.add_latency("readv", self._get_current_time() - start) return {} datavs = {} for sharenum_s in os.listdir(bucketdir): @@ -689,7 +691,7 @@ class StorageServer(service.MultiService, Referenceable): datavs[sharenum] = msf.readv(readv) log.msg("returning shares %s" % (list(datavs.keys()),), facility="tahoe.storage", level=log.NOISY, parent=lp) - self.add_latency("readv", time.time() - start) + self.add_latency("readv", self._get_current_time() - start) return datavs def remote_advise_corrupt_share(self, share_type, storage_index, shnum, From d599568c7988b2769d72c9e63ab4a8368fb9867c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 19 Aug 2021 13:56:13 -0400 Subject: [PATCH 075/269] Tests validating that the operation for adding a lease will renew the lease instead if it already exists. --- src/allmydata/storage/server.py | 5 +- src/allmydata/test/test_storage.py | 116 ++++++++++++++++++++++------- 2 files changed, 92 insertions(+), 29 deletions(-) diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py index fb0cf1201..4615c9ec9 100644 --- a/src/allmydata/storage/server.py +++ b/src/allmydata/storage/server.py @@ -49,6 +49,9 @@ from allmydata.storage.expirer import LeaseCheckingCrawler NUM_RE=re.compile("^[0-9]+$") +# Number of seconds to add to expiration time on lease renewal: +DEFAULT_RENEWAL_TIME = 31 * 24 * 60 * 60 + @implementer(RIStorageServer, IStatsProducer) class StorageServer(service.MultiService, Referenceable): @@ -279,7 +282,7 @@ class StorageServer(service.MultiService, Referenceable): # goes into the share files themselves. It could also be put into a # separate database. Note that the lease should not be added until # the BucketWriter has been closed. - expire_time = self._get_current_time() + 31*24*60*60 + expire_time = self._get_current_time() + DEFAULT_RENEWAL_TIME lease_info = LeaseInfo(owner_num, renew_secret, cancel_secret, expire_time, self.my_nodeid) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index d4de0c18b..8ad99a7ab 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -24,11 +24,12 @@ import gc from twisted.trial import unittest from twisted.internet import defer +from twisted.internet.task import Clock import itertools from allmydata import interfaces from allmydata.util import fileutil, hashutil, base32 -from allmydata.storage.server import StorageServer +from allmydata.storage.server import StorageServer, DEFAULT_RENEWAL_TIME from allmydata.storage.shares import get_share_file from allmydata.storage.mutable import MutableShareFile from allmydata.storage.immutable import BucketWriter, BucketReader, ShareFile @@ -168,7 +169,7 @@ class Bucket(unittest.TestCase): assert len(renewsecret) == 32 cancelsecret = b'THIS LETS ME KILL YOUR FILE HAHA' assert len(cancelsecret) == 32 - expirationtime = struct.pack('>L', 60*60*24*31) # 31 days in seconds + expirationtime = struct.pack('>L', DEFAULT_RENEWAL_TIME) # 31 days in seconds lease_data = ownernumber + renewsecret + cancelsecret + expirationtime @@ -354,10 +355,11 @@ class Server(unittest.TestCase): basedir = os.path.join("storage", "Server", name) return basedir - def create(self, name, reserved_space=0, klass=StorageServer): + def create(self, name, reserved_space=0, klass=StorageServer, get_current_time=time.time): workdir = self.workdir(name) ss = klass(workdir, b"\x00" * 20, reserved_space=reserved_space, - stats_provider=FakeStatsProvider()) + stats_provider=FakeStatsProvider(), + get_current_time=get_current_time) ss.setServiceParent(self.sparent) return ss @@ -646,6 +648,25 @@ class Server(unittest.TestCase): f2 = open(filename, "rb") self.failUnlessEqual(f2.read(5), b"start") + def create_bucket(self, ss, storage_index, expected_already=0, expected_writers=5): + """ + Given a StorageServer, create a bucket and return renewal and + cancellation secrets. + """ + canary = FakeCanary() + sharenums = list(range(5)) + size = 100 + + # Creating a bucket also creates a lease: + rs, cs = (hashutil.tagged_hash(b"blah", b"%d" % next(self._lease_secret)), + hashutil.tagged_hash(b"blah", b"%d" % next(self._lease_secret))) + already, writers = ss.remote_allocate_buckets(storage_index, rs, cs, + sharenums, size, canary) + self.failUnlessEqual(len(already), expected_already) + self.failUnlessEqual(len(writers), expected_writers) + for wb in writers.values(): + wb.remote_close() + return rs, cs def test_leases(self): ss = self.create("test_leases") @@ -653,34 +674,16 @@ class Server(unittest.TestCase): sharenums = list(range(5)) size = 100 - rs0,cs0 = (hashutil.tagged_hash(b"blah", b"%d" % next(self._lease_secret)), - hashutil.tagged_hash(b"blah", b"%d" % next(self._lease_secret))) - already,writers = ss.remote_allocate_buckets(b"si0", rs0, cs0, - sharenums, size, canary) - self.failUnlessEqual(len(already), 0) - self.failUnlessEqual(len(writers), 5) - for wb in writers.values(): - wb.remote_close() - + # Create a bucket: + rs0, cs0 = self.create_bucket(ss, b"si0") leases = list(ss.get_leases(b"si0")) self.failUnlessEqual(len(leases), 1) self.failUnlessEqual(set([l.renew_secret for l in leases]), set([rs0])) - rs1,cs1 = (hashutil.tagged_hash(b"blah", b"%d" % next(self._lease_secret)), - hashutil.tagged_hash(b"blah", b"%d" % next(self._lease_secret))) - already,writers = ss.remote_allocate_buckets(b"si1", rs1, cs1, - sharenums, size, canary) - for wb in writers.values(): - wb.remote_close() + rs1, cs1 = self.create_bucket(ss, b"si1") # take out a second lease on si1 - rs2,cs2 = (hashutil.tagged_hash(b"blah", b"%d" % next(self._lease_secret)), - hashutil.tagged_hash(b"blah", b"%d" % next(self._lease_secret))) - already,writers = ss.remote_allocate_buckets(b"si1", rs2, cs2, - sharenums, size, canary) - self.failUnlessEqual(len(already), 5) - self.failUnlessEqual(len(writers), 0) - + rs2, cs2 = self.create_bucket(ss, b"si1", 5, 0) leases = list(ss.get_leases(b"si1")) self.failUnlessEqual(len(leases), 2) self.failUnlessEqual(set([l.renew_secret for l in leases]), set([rs1, rs2])) @@ -741,6 +744,27 @@ class Server(unittest.TestCase): leases = list(ss.get_leases(b"si3")) self.failUnlessEqual(len(leases), 2) + def test_immutable_add_lease_renews(self): + """ + Adding a lease on an already leased immutable just renews it. + """ + clock = Clock() + clock.advance(123) + ss = self.create("test_immutable_add_lease_renews", get_current_time=clock.seconds) + + # Start out with single lease created with bucket: + renewal_secret, cancel_secret = self.create_bucket(ss, b"si0") + [lease] = ss.get_leases(b"si0") + self.assertEqual(lease.expiration_time, 123 + DEFAULT_RENEWAL_TIME) + + # Time passes: + clock.advance(123456) + + # Adding a lease with matching renewal secret just renews it: + ss.remote_add_lease(b"si0", renewal_secret, cancel_secret) + [lease] = ss.get_leases(b"si0") + self.assertEqual(lease.expiration_time, 123 + 123456 + DEFAULT_RENEWAL_TIME) + def test_have_shares(self): """By default the StorageServer has no shares.""" workdir = self.workdir("test_have_shares") @@ -840,9 +864,10 @@ class MutableServer(unittest.TestCase): basedir = os.path.join("storage", "MutableServer", name) return basedir - def create(self, name): + def create(self, name, get_current_time=time.time): workdir = self.workdir(name) - ss = StorageServer(workdir, b"\x00" * 20) + ss = StorageServer(workdir, b"\x00" * 20, + get_current_time=get_current_time) ss.setServiceParent(self.sparent) return ss @@ -1379,6 +1404,41 @@ class MutableServer(unittest.TestCase): {0: ([], [(500, b"make me really bigger")], None)}, []) self.compare_leases_without_timestamps(all_leases, list(s0.get_leases())) + def test_mutable_add_lease_renews(self): + """ + Adding a lease on an already leased mutable just renews it. + """ + clock = Clock() + clock.advance(235) + ss = self.create("test_mutable_add_lease_renews", + get_current_time=clock.seconds) + def secrets(n): + return ( self.write_enabler(b"we1"), + self.renew_secret(b"we1-%d" % n), + self.cancel_secret(b"we1-%d" % n) ) + data = b"".join([ (b"%d" % i) * 10 for i in range(10) ]) + write = ss.remote_slot_testv_and_readv_and_writev + read = ss.remote_slot_readv + write_enabler, renew_secret, cancel_secret = secrets(0) + rc = write(b"si1", (write_enabler, renew_secret, cancel_secret), + {0: ([], [(0,data)], None)}, []) + self.failUnlessEqual(rc, (True, {})) + + bucket_dir = os.path.join(self.workdir("test_mutable_add_lease_renews"), + "shares", storage_index_to_dir(b"si1")) + s0 = MutableShareFile(os.path.join(bucket_dir, "0")) + [lease] = s0.get_leases() + self.assertEqual(lease.expiration_time, 235 + DEFAULT_RENEWAL_TIME) + + # Time passes... + clock.advance(835) + + # Adding a lease renews it: + ss.remote_add_lease(b"si1", renew_secret, cancel_secret) + [lease] = s0.get_leases() + self.assertEqual(lease.expiration_time, + 235 + 835 + DEFAULT_RENEWAL_TIME) + def test_remove(self): ss = self.create("test_remove") self.allocate(ss, b"si1", b"we1", next(self._lease_secret), From 00163690258c5dd8242a470775211109bcdffa37 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 19 Aug 2021 16:07:03 -0400 Subject: [PATCH 076/269] News file. --- newsfragments/3773.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3773.minor diff --git a/newsfragments/3773.minor b/newsfragments/3773.minor new file mode 100644 index 000000000..e69de29bb From 59fab99d9d52c4d172087256f6cba1dd5b9f9a7e Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 19 Aug 2021 16:40:45 -0400 Subject: [PATCH 077/269] Nothing uses RIStorageServer.renew_lease, so removing it is simple. --- src/allmydata/interfaces.py | 24 ------------------------ src/allmydata/storage_client.py | 11 ----------- 2 files changed, 35 deletions(-) diff --git a/src/allmydata/interfaces.py b/src/allmydata/interfaces.py index 2e055a888..1c64bce8a 100644 --- a/src/allmydata/interfaces.py +++ b/src/allmydata/interfaces.py @@ -154,25 +154,9 @@ class RIStorageServer(RemoteInterface): """ return Any() # returns None now, but future versions might change - def renew_lease(storage_index=StorageIndex, renew_secret=LeaseRenewSecret): - """ - Renew the lease on a given bucket, resetting the timer to 31 days. - Some networks will use this, some will not. If there is no bucket for - the given storage_index, IndexError will be raised. - - For mutable shares, if the given renew_secret does not match an - existing lease, IndexError will be raised with a note listing the - server-nodeids on the existing leases, so leases on migrated shares - can be renewed. For immutable shares, IndexError (without the note) - will be raised. - """ - return Any() - def get_buckets(storage_index=StorageIndex): return DictOf(int, RIBucketReader, maxKeys=MAX_BUCKETS) - - def slot_readv(storage_index=StorageIndex, shares=ListOf(int), readv=ReadVector): """Read a vector from the numbered shares associated with the given @@ -343,14 +327,6 @@ class IStorageServer(Interface): :see: ``RIStorageServer.add_lease`` """ - def renew_lease( - storage_index, - renew_secret, - ): - """ - :see: ``RIStorageServer.renew_lease`` - """ - def get_buckets( storage_index, ): diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index b0bcc6835..22dd09bcb 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -965,17 +965,6 @@ class _StorageServer(object): cancel_secret, ) - def renew_lease( - self, - storage_index, - renew_secret, - ): - return self._rref.callRemote( - "renew_lease", - storage_index, - renew_secret, - ) - def get_buckets( self, storage_index, From 442d61da7db3776955241b4c85af3e52af363ba9 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 19 Aug 2021 16:44:05 -0400 Subject: [PATCH 078/269] Get rid of separate renewal of leases in HTTP API. --- docs/proposed/http-storage-node-protocol.rst | 28 +------------------- 1 file changed, 1 insertion(+), 27 deletions(-) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index 41a0a0fea..d1f63afed 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -369,7 +369,7 @@ For example:: ``PUT /v1/lease/:storage_index`` !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -Create a new lease that applies to all shares for the given storage index. +Either renew or create a new lease that applies to all shares for the given storage index. The details of the lease are encoded in the request body. For example:: @@ -394,37 +394,11 @@ Several behaviors here are blindly copied from the Foolscap-based storage server * There is a cancel secret but there is no API to use it to cancel a lease. * The lease period is hard-coded at 31 days. * There is no way to differentiate between success and an unknown **storage index**. -* There are separate **add** and **renew** lease APIs. These are not necessarily ideal behaviors but they are adopted to avoid any *semantic* changes between the Foolscap- and HTTP-based protocols. It is expected that some or all of these behaviors may change in a future revision of the HTTP-based protocol. -``POST /v1/lease/:storage_index`` -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - -Renew an existing lease for all shares for the given storage index. -The details of the lease are encoded in the request body. -For example:: - - {"renew-secret": "abcd"} - -If there are no shares for the given ``storage_index`` -then ``NOT FOUND`` is returned. - -If there is no lease with a matching ``renew-secret`` value on the given storage index -then ``NOT FOUND`` is returned. -In this case, -if the storage index refers to mutable data -then the response also includes a list of nodeids where the lease can be renewed. -For example:: - - {"nodeids": ["aaa...", "bbb..."]} - -Othewise, -the matching lease's expiration time is changed to be 31 days from the time of this operation -and ``NO CONTENT`` is returned. - Immutable --------- From 1b5a3c9cb17b987e687dfe88910cd500b263afdd Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 19 Aug 2021 16:49:01 -0400 Subject: [PATCH 079/269] Bad merge, I think. --- docs/proposed/http-storage-node-protocol.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index 99840620f..30ccaa9d2 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -400,7 +400,6 @@ Several behaviors here are blindly copied from the Foolscap-based storage server * There is a cancel secret but there is no API to use it to cancel a lease (see ticket:3768). * The lease period is hard-coded at 31 days. -* There is no way to differentiate between success and an unknown **storage index**. These are not necessarily ideal behaviors but they are adopted to avoid any *semantic* changes between the Foolscap- and HTTP-based protocols. From 370d1ddafe0a7267a456d6107b854a08fee3af05 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 20 Aug 2021 08:20:24 -0400 Subject: [PATCH 080/269] Fix flake. --- src/allmydata/test/test_storage.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index 8ad99a7ab..32c2b25b8 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -1418,7 +1418,6 @@ class MutableServer(unittest.TestCase): self.cancel_secret(b"we1-%d" % n) ) data = b"".join([ (b"%d" % i) * 10 for i in range(10) ]) write = ss.remote_slot_testv_and_readv_and_writev - read = ss.remote_slot_readv write_enabler, renew_secret, cancel_secret = secrets(0) rc = write(b"si1", (write_enabler, renew_secret, cancel_secret), {0: ([], [(0,data)], None)}, []) From 11331ddf8209249ebba6280e17db52d1534b3459 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 20 Aug 2021 11:17:20 -0400 Subject: [PATCH 081/269] Update examples to lack of separate renewal endpoint. --- docs/proposed/http-storage-node-protocol.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index 30ccaa9d2..dd696c340 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -650,8 +650,8 @@ Immutable Data #. Renew the lease on all immutable shares in bucket ``AAAAAAAAAAAAAAAA``:: - POST /v1/lease/AAAAAAAAAAAAAAAA - {"renew-secret": "efgh"} + PUT /v1/lease/AAAAAAAAAAAAAAAA + {"renew-secret": "efgh", "cancel-secret": "ijkl"} 204 NO CONTENT @@ -731,8 +731,8 @@ Mutable Data #. Renew the lease on previously uploaded mutable share in slot ``BBBBBBBBBBBBBBBB``:: - POST /v1/lease/BBBBBBBBBBBBBBBB - {"renew-secret": "efgh"} + PUT /v1/lease/BBBBBBBBBBBBBBBB + {"renew-secret": "efgh", "cancel-secret": "ijkl"} 204 NO CONTENT From 117befd8986596f1b6903a68076aad75da4a0b7c Mon Sep 17 00:00:00 2001 From: fenn-cs Date: Mon, 23 Aug 2021 02:34:32 +0100 Subject: [PATCH 082/269] detect all port 0 configs Signed-off-by: fenn-cs --- newsfragments/3563.minor | 1 + src/allmydata/node.py | 2 +- src/allmydata/test/test_node.py | 17 +++++++++++------ 3 files changed, 13 insertions(+), 7 deletions(-) create mode 100644 newsfragments/3563.minor diff --git a/newsfragments/3563.minor b/newsfragments/3563.minor new file mode 100644 index 000000000..038b50995 --- /dev/null +++ b/newsfragments/3563.minor @@ -0,0 +1 @@ +detect all port 0 configs \ No newline at end of file diff --git a/src/allmydata/node.py b/src/allmydata/node.py index 4dcb7cc76..5a6f8c66f 100644 --- a/src/allmydata/node.py +++ b/src/allmydata/node.py @@ -793,7 +793,7 @@ def _tub_portlocation(config, get_local_addresses_sync, allocate_tcp_port): tubport = _convert_tub_port(cfg_tubport) for port in tubport.split(","): - if port in ("0", "tcp:0"): + if port in ("0", "tcp:0", "tcp:port=0", "tcp:0:interface=127.0.0.1"): raise PortAssignmentRequired() if cfg_location is None: diff --git a/src/allmydata/test/test_node.py b/src/allmydata/test/test_node.py index e44fd5743..cf5fa27f3 100644 --- a/src/allmydata/test/test_node.py +++ b/src/allmydata/test/test_node.py @@ -6,7 +6,7 @@ from __future__ import division from __future__ import print_function from __future__ import unicode_literals -from future.utils import PY2, native_str +from future.utils import PY2 if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 @@ -530,6 +530,14 @@ def _stub_allocate_tcp_port(): """ return 999 +def _stub_none(): + """ + A function like ``_stub_allocate_tcp`` or ``_stub_get_local_addresses_sync`` + but that return an empty list since ``allmydata.node._tub_portlocation`` requires a + callable for paramter 1 and 2 counting from 0. + """ + return [] + class TestMissingPorts(unittest.TestCase): """ @@ -550,7 +558,7 @@ class TestMissingPorts(unittest.TestCase): ) config = config_from_string(self.basedir, "portnum", config_data) with self.assertRaises(PortAssignmentRequired): - _tub_portlocation(config, None, None) + _tub_portlocation(config, _stub_none, _stub_none) def test_listen_on_zero_with_host(self): """ @@ -563,10 +571,7 @@ class TestMissingPorts(unittest.TestCase): ) config = config_from_string(self.basedir, "portnum", config_data) with self.assertRaises(PortAssignmentRequired): - _tub_portlocation(config, None, None) - test_listen_on_zero_with_host.todo = native_str( # type: ignore - "https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3563" - ) + _tub_portlocation(config, _stub_none, _stub_none) def test_parsing_tcp(self): """ From 398fe537e8da2e8d8e4eaf5e9bd145cbd32025e1 Mon Sep 17 00:00:00 2001 From: fenn-cs Date: Mon, 23 Aug 2021 02:48:24 +0100 Subject: [PATCH 083/269] added \n in newfragment file Signed-off-by: fenn-cs --- newsfragments/3563.minor | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/newsfragments/3563.minor b/newsfragments/3563.minor index 038b50995..ea08fda56 100644 --- a/newsfragments/3563.minor +++ b/newsfragments/3563.minor @@ -1 +1 @@ -detect all port 0 configs \ No newline at end of file +detect all port 0 configs From 7a88b5ddae5497f52421deb2e852971809a62b45 Mon Sep 17 00:00:00 2001 From: fenn-cs Date: Tue, 24 Aug 2021 13:05:11 +0100 Subject: [PATCH 084/269] removed unnecessary comment in minor newsfragment Signed-off-by: fenn-cs --- newsfragments/3563.minor | 1 - 1 file changed, 1 deletion(-) diff --git a/newsfragments/3563.minor b/newsfragments/3563.minor index ea08fda56..e69de29bb 100644 --- a/newsfragments/3563.minor +++ b/newsfragments/3563.minor @@ -1 +0,0 @@ -detect all port 0 configs From 3e0dc9449757054fb3a3fe3ffad8164c5fdb265d Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 25 Aug 2021 13:06:10 -0400 Subject: [PATCH 085/269] Annotate the two fakes that (at least partially) implement RIStorageServer, so they're easier to find. --- src/allmydata/test/mutable/util.py | 8 +++++++- src/allmydata/test/test_upload.py | 8 ++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/src/allmydata/test/mutable/util.py b/src/allmydata/test/mutable/util.py index 62e8d7295..7e3bd3ec7 100644 --- a/src/allmydata/test/mutable/util.py +++ b/src/allmydata/test/mutable/util.py @@ -96,8 +96,14 @@ class FakeStorage(object): shares[shnum] = f.getvalue() +# This doesn't actually implement the whole interface, but adding a commented +# interface implementation annotation for grepping purposes. +#@implementer(RIStorageServer) class FakeStorageServer(object): - + """ + A fake Foolscap remote object, implemented by overriding callRemote() to + call local methods. + """ def __init__(self, peerid, storage): self.peerid = peerid self.storage = storage diff --git a/src/allmydata/test/test_upload.py b/src/allmydata/test/test_upload.py index fc9bfd697..8d5435e88 100644 --- a/src/allmydata/test/test_upload.py +++ b/src/allmydata/test/test_upload.py @@ -122,7 +122,15 @@ class SetDEPMixin(object): } self.node.encoding_params = p + +# This doesn't actually implement the whole interface, but adding a commented +# interface implementation annotation for grepping purposes. +#@implementer(RIStorageServer) class FakeStorageServer(object): + """ + A fake Foolscap remote object, implemented by overriding callRemote() to + call local methods. + """ def __init__(self, mode, reactor=None): self.mode = mode self.allocated = [] From 52cb25070142f6648c759996cdafe992ae37d94a Mon Sep 17 00:00:00 2001 From: Florian Sesser Date: Fri, 27 Aug 2021 16:42:23 +0000 Subject: [PATCH 086/269] This is the handler we need to create. --- src/allmydata/web/status.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/allmydata/web/status.py b/src/allmydata/web/status.py index 158d897f9..8c78b1156 100644 --- a/src/allmydata/web/status.py +++ b/src/allmydata/web/status.py @@ -1551,6 +1551,21 @@ class Statistics(MultiFormatResource): req.setHeader("content-type", "text/plain") return json.dumps(stats, indent=1) + "\n" + @render_exception + def render_OPENMETRICS(self, req): + req.setHeader("content-type", "application/openmetrics-text") + return "3. occurence. This should be the one.\n" + + # @render_exception + # def render_OPENMETRICS(self, req): + # req.setHeader("content-type", "text/plain") + # if self._helper: + # stats = self._helper.get_stats() + # import pprint + # return pprint.PrettyPrinter().pprint(stats) + "\n" + # return "uh oh\n" + + class StatisticsElement(Element): loader = XMLFile(FilePath(__file__).sibling("statistics.xhtml")) From 342a1c2c31ead596cb705a30e69282ad28df9932 Mon Sep 17 00:00:00 2001 From: fenn-cs Date: Sun, 29 Aug 2021 16:10:30 +0100 Subject: [PATCH 087/269] Python 3 support complete, so removed warning Signed-off-by: fenn-cs --- newsfragments/3781.minor | 0 src/allmydata/scripts/runner.py | 5 ----- src/allmydata/test/test_system.py | 8 ++------ 3 files changed, 2 insertions(+), 11 deletions(-) create mode 100644 newsfragments/3781.minor diff --git a/newsfragments/3781.minor b/newsfragments/3781.minor new file mode 100644 index 000000000..e69de29bb diff --git a/src/allmydata/scripts/runner.py b/src/allmydata/scripts/runner.py index 7f7f88bf6..a42b30624 100644 --- a/src/allmydata/scripts/runner.py +++ b/src/allmydata/scripts/runner.py @@ -183,13 +183,8 @@ def _maybe_enable_eliot_logging(options, reactor): # Pass on the options so we can dispatch the subcommand. return options -PYTHON_3_WARNING = ("Support for Python 3 is an incomplete work-in-progress." - " Use at your own risk.") def run(): - if six.PY3: - print(PYTHON_3_WARNING, file=sys.stderr) - if sys.platform == "win32": from allmydata.windows.fixups import initialize initialize() diff --git a/src/allmydata/test/test_system.py b/src/allmydata/test/test_system.py index 627b6ef29..d61d6990b 100644 --- a/src/allmydata/test/test_system.py +++ b/src/allmydata/test/test_system.py @@ -43,7 +43,6 @@ from allmydata.monitor import Monitor from allmydata.mutable.common import NotWriteableError from allmydata.mutable import layout as mutable_layout from allmydata.mutable.publish import MutableData -from allmydata.scripts.runner import PYTHON_3_WARNING from foolscap.api import DeadReferenceError, fireEventually, flushEventualQueue from twisted.python.failure import Failure @@ -2632,18 +2631,16 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase): newargs = ["--node-directory", self.getdir("client0"), verb] + list(args) return self.run_bintahoe(newargs, stdin=stdin, env=env) - def _check_succeeded(res, check_stderr=True): + def _check_succeeded(res): out, err, rc_or_sig = res self.failUnlessEqual(rc_or_sig, 0, str(res)) - if check_stderr: - self.assertIn(err.strip(), (b"", PYTHON_3_WARNING.encode("ascii"))) d.addCallback(_run_in_subprocess, "create-alias", "newalias") d.addCallback(_check_succeeded) STDIN_DATA = b"This is the file to upload from stdin." d.addCallback(_run_in_subprocess, "put", "-", "newalias:tahoe-file", stdin=STDIN_DATA) - d.addCallback(_check_succeeded, check_stderr=False) + d.addCallback(_check_succeeded) def _mv_with_http_proxy(ign): env = os.environ @@ -2656,7 +2653,6 @@ class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase): def _check_ls(res): out, err, rc_or_sig = res self.failUnlessEqual(rc_or_sig, 0, str(res)) - self.assertIn(err.strip(), (b"", PYTHON_3_WARNING.encode("ascii"))) self.failUnlessIn(b"tahoe-moved", out) self.failIfIn(b"tahoe-file", out) d.addCallback(_check_ls) From 7d32335353e4960b3be9d7b8ad91214e493e9b4f Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 30 Aug 2021 16:49:11 -0400 Subject: [PATCH 088/269] Clarify and remove duplication. --- src/allmydata/storage/server.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py index 4615c9ec9..56b47ae89 100644 --- a/src/allmydata/storage/server.py +++ b/src/allmydata/storage/server.py @@ -49,7 +49,8 @@ from allmydata.storage.expirer import LeaseCheckingCrawler NUM_RE=re.compile("^[0-9]+$") -# Number of seconds to add to expiration time on lease renewal: +# Number of seconds to add to expiration time on lease renewal. +# For now it's not actually configurable, but maybe someday. DEFAULT_RENEWAL_TIME = 31 * 24 * 60 * 60 @@ -358,7 +359,7 @@ class StorageServer(service.MultiService, Referenceable): owner_num=1): start = self._get_current_time() self.count("add-lease") - new_expire_time = self._get_current_time() + 31*24*60*60 + new_expire_time = self._get_current_time() + DEFAULT_RENEWAL_TIME lease_info = LeaseInfo(owner_num, renew_secret, cancel_secret, new_expire_time, self.my_nodeid) @@ -370,7 +371,7 @@ class StorageServer(service.MultiService, Referenceable): def remote_renew_lease(self, storage_index, renew_secret): start = self._get_current_time() self.count("renew") - new_expire_time = self._get_current_time() + 31*24*60*60 + new_expire_time = self._get_current_time() + DEFAULT_RENEWAL_TIME found_buckets = False for sf in self._iter_share_files(storage_index): found_buckets = True @@ -568,7 +569,7 @@ class StorageServer(service.MultiService, Referenceable): :return LeaseInfo: Information for a new lease for a share. """ ownerid = 1 # TODO - expire_time = self._get_current_time() + 31*24*60*60 # one month + expire_time = self._get_current_time() + DEFAULT_RENEWAL_TIME lease_info = LeaseInfo(ownerid, renew_secret, cancel_secret, expire_time, self.my_nodeid) From e408322c3da5c6b93d5eab5755985c0b4323c3ae Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 30 Aug 2021 16:51:36 -0400 Subject: [PATCH 089/269] Use the correct APIs. --- src/allmydata/test/test_storage.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index 32c2b25b8..4b76fed31 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -386,8 +386,8 @@ class Server(unittest.TestCase): self.failUnlessIn(b'available-space', sv1) def allocate(self, ss, storage_index, sharenums, size, canary=None): - renew_secret = hashutil.tagged_hash(b"blah", b"%d" % next(self._lease_secret)) - cancel_secret = hashutil.tagged_hash(b"blah", b"%d" % next(self._lease_secret)) + renew_secret = hashutil.my_renewal_secret_hash(b"%d" % next(self._lease_secret)) + cancel_secret = hashutil.my_cancel_secret_hash(b"%d" % next(self._lease_secret)) if not canary: canary = FakeCanary() return ss.remote_allocate_buckets(storage_index, @@ -658,8 +658,8 @@ class Server(unittest.TestCase): size = 100 # Creating a bucket also creates a lease: - rs, cs = (hashutil.tagged_hash(b"blah", b"%d" % next(self._lease_secret)), - hashutil.tagged_hash(b"blah", b"%d" % next(self._lease_secret))) + rs, cs = (hashutil.my_renewal_secret_hash(b"%d" % next(self._lease_secret)), + hashutil.my_cancel_secret_hash(b"%d" % next(self._lease_secret))) already, writers = ss.remote_allocate_buckets(storage_index, rs, cs, sharenums, size, canary) self.failUnlessEqual(len(already), expected_already) @@ -689,8 +689,8 @@ class Server(unittest.TestCase): self.failUnlessEqual(set([l.renew_secret for l in leases]), set([rs1, rs2])) # and a third lease, using add-lease - rs2a,cs2a = (hashutil.tagged_hash(b"blah", b"%d" % next(self._lease_secret)), - hashutil.tagged_hash(b"blah", b"%d" % next(self._lease_secret))) + rs2a,cs2a = (hashutil.my_renewal_secret_hash(b"%d" % next(self._lease_secret)), + hashutil.my_cancel_secret_hash(b"%d" % next(self._lease_secret))) ss.remote_add_lease(b"si1", rs2a, cs2a) leases = list(ss.get_leases(b"si1")) self.failUnlessEqual(len(leases), 3) @@ -718,10 +718,10 @@ class Server(unittest.TestCase): "ss should not have a 'remote_cancel_lease' method/attribute") # test overlapping uploads - rs3,cs3 = (hashutil.tagged_hash(b"blah", b"%d" % next(self._lease_secret)), - hashutil.tagged_hash(b"blah", b"%d" % next(self._lease_secret))) - rs4,cs4 = (hashutil.tagged_hash(b"blah", b"%d" % next(self._lease_secret)), - hashutil.tagged_hash(b"blah", b"%d" % next(self._lease_secret))) + rs3,cs3 = (hashutil.my_renewal_secret_hash(b"%d" % next(self._lease_secret)), + hashutil.my_cancel_secret_hash(b"%d" % next(self._lease_secret))) + rs4,cs4 = (hashutil.my_renewal_secret_hash(b"%d" % next(self._lease_secret)), + hashutil.my_cancel_secret_hash(b"%d" % next(self._lease_secret))) already,writers = ss.remote_allocate_buckets(b"si3", rs3, cs3, sharenums, size, canary) self.failUnlessEqual(len(already), 0) From e6803670d1aae43d349af240b586d4aefffef86e Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 30 Aug 2021 16:54:44 -0400 Subject: [PATCH 090/269] Improve explanations. --- src/allmydata/test/test_storage.py | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index 4b76fed31..bd0ab80f3 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -648,10 +648,12 @@ class Server(unittest.TestCase): f2 = open(filename, "rb") self.failUnlessEqual(f2.read(5), b"start") - def create_bucket(self, ss, storage_index, expected_already=0, expected_writers=5): + def create_bucket_5_shares( + self, ss, storage_index, expected_already=0, expected_writers=5 + ): """ - Given a StorageServer, create a bucket and return renewal and - cancellation secrets. + Given a StorageServer, create a bucket with 5 shares and return renewal + and cancellation secrets. """ canary = FakeCanary() sharenums = list(range(5)) @@ -675,15 +677,15 @@ class Server(unittest.TestCase): size = 100 # Create a bucket: - rs0, cs0 = self.create_bucket(ss, b"si0") + rs0, cs0 = self.create_bucket_5_shares(ss, b"si0") leases = list(ss.get_leases(b"si0")) self.failUnlessEqual(len(leases), 1) self.failUnlessEqual(set([l.renew_secret for l in leases]), set([rs0])) - rs1, cs1 = self.create_bucket(ss, b"si1") + rs1, cs1 = self.create_bucket_5_shares(ss, b"si1") # take out a second lease on si1 - rs2, cs2 = self.create_bucket(ss, b"si1", 5, 0) + rs2, cs2 = self.create_bucket_5_shares(ss, b"si1", 5, 0) leases = list(ss.get_leases(b"si1")) self.failUnlessEqual(len(leases), 2) self.failUnlessEqual(set([l.renew_secret for l in leases]), set([rs1, rs2])) @@ -746,14 +748,15 @@ class Server(unittest.TestCase): def test_immutable_add_lease_renews(self): """ - Adding a lease on an already leased immutable just renews it. + Adding a lease on an already leased immutable with the same secret just + renews it. """ clock = Clock() clock.advance(123) ss = self.create("test_immutable_add_lease_renews", get_current_time=clock.seconds) # Start out with single lease created with bucket: - renewal_secret, cancel_secret = self.create_bucket(ss, b"si0") + renewal_secret, cancel_secret = self.create_bucket_5_shares(ss, b"si0") [lease] = ss.get_leases(b"si0") self.assertEqual(lease.expiration_time, 123 + DEFAULT_RENEWAL_TIME) @@ -1406,7 +1409,8 @@ class MutableServer(unittest.TestCase): def test_mutable_add_lease_renews(self): """ - Adding a lease on an already leased mutable just renews it. + Adding a lease on an already leased mutable with the same secret just + renews it. """ clock = Clock() clock.advance(235) From 8202256fa2640b193022f58772b903ae7f730460 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 31 Aug 2021 09:58:03 -0400 Subject: [PATCH 091/269] Add newsfragment --- newsfragments/3782.documentation | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/3782.documentation diff --git a/newsfragments/3782.documentation b/newsfragments/3782.documentation new file mode 100644 index 000000000..5e5cecc13 --- /dev/null +++ b/newsfragments/3782.documentation @@ -0,0 +1 @@ +tahoe-dev mailing list is now at tahoe-dev@lists.tahoe-lafs.org. From 65a1040fe884aee4f075bc6ccc0ed2a9e2dc26cc Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 31 Aug 2021 09:58:43 -0400 Subject: [PATCH 092/269] Update references to the mailing list New list is tahoe-dev@lists.tahoe-lafs.org, list info page is at https://lists.tahoe-lafs.org/mailman/listinfo/tahoe-dev, and list archives are now at https://lists.tahoe-lafs.org/pipermail/tahoe-dev/. Sadly message numbers in list archive seem to have changed, so updating references to list archive is not as simple as prefixing `list.` --- NEWS.rst | 2 +- README.rst | 2 +- docs/Installation/install-tahoe.rst | 2 +- docs/historical/historical_known_issues.txt | 2 +- docs/man/man1/tahoe.1 | 2 +- docs/release-checklist.rst | 4 ++-- docs/running.rst | 2 +- misc/python3/depgraph.sh | 2 +- relnotes.txt | 2 +- setup.py | 2 +- 10 files changed, 11 insertions(+), 11 deletions(-) diff --git a/NEWS.rst b/NEWS.rst index 88d231826..d70fac6b5 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -1188,7 +1188,7 @@ Precautions when Upgrading .. _`#1915`: https://tahoe-lafs.org/trac/tahoe-lafs/ticket/1915 .. _`#1926`: https://tahoe-lafs.org/trac/tahoe-lafs/ticket/1926 .. _`message to the tahoe-dev mailing list`: - https://tahoe-lafs.org/pipermail/tahoe-dev/2013-March/008096.html + https://lists.tahoe-lafs.org/pipermail/tahoe-dev/2013-March/008096.html Release 1.9.2 (2012-07-03) diff --git a/README.rst b/README.rst index 2cc6e38eb..e58211601 100644 --- a/README.rst +++ b/README.rst @@ -76,7 +76,7 @@ Get involved with the Tahoe-LAFS community: - Join our `weekly conference calls `__ with core developers and interested community members. -- Subscribe to `the tahoe-dev mailing list `__, the community forum for discussion of Tahoe-LAFS design, implementation, and usage. +- Subscribe to `the tahoe-dev mailing list `__, the community forum for discussion of Tahoe-LAFS design, implementation, and usage. 🤗 Contributing --------------- diff --git a/docs/Installation/install-tahoe.rst b/docs/Installation/install-tahoe.rst index c8b0b521e..2fe47f4a8 100644 --- a/docs/Installation/install-tahoe.rst +++ b/docs/Installation/install-tahoe.rst @@ -65,4 +65,4 @@ If you are working on MacOS or a Linux distribution which does not have Tahoe-LA If you are looking to hack on the source code or run pre-release code, we recommend you install Tahoe-LAFS on a `virtualenv` instance. To learn more, see :doc:`install-on-linux`. -You can always write to the `tahoe-dev mailing list `_ or chat on the `Libera.chat IRC `_ if you are not able to get Tahoe-LAFS up and running on your deployment. +You can always write to the `tahoe-dev mailing list `_ or chat on the `Libera.chat IRC `_ if you are not able to get Tahoe-LAFS up and running on your deployment. diff --git a/docs/historical/historical_known_issues.txt b/docs/historical/historical_known_issues.txt index 9d4e1d427..8edab51a8 100644 --- a/docs/historical/historical_known_issues.txt +++ b/docs/historical/historical_known_issues.txt @@ -177,7 +177,7 @@ mutable files, you may be able to avoid the potential for "rollback" failure. A future version of Tahoe will include a fix for this issue. Here is -[https://tahoe-lafs.org/pipermail/tahoe-dev/2008-May/000630.html the +[https://lists.tahoe-lafs.org/pipermail/tahoe-dev/2008-May/000628.html the mailing list discussion] about how that future version will work. diff --git a/docs/man/man1/tahoe.1 b/docs/man/man1/tahoe.1 index 113f6a311..ab3d3c4c5 100644 --- a/docs/man/man1/tahoe.1 +++ b/docs/man/man1/tahoe.1 @@ -268,7 +268,7 @@ For known security issues see .PP Tahoe-LAFS home page: .PP -tahoe-dev mailing list: +tahoe-dev mailing list: .SH COPYRIGHT .PP Copyright \@ 2006\[en]2013 The Tahoe-LAFS Software Foundation diff --git a/docs/release-checklist.rst b/docs/release-checklist.rst index a5761c1c7..da1bbe16f 100644 --- a/docs/release-checklist.rst +++ b/docs/release-checklist.rst @@ -178,8 +178,8 @@ Announcing the Release Candidate ```````````````````````````````` The release-candidate should be announced by posting to the -mailing-list (tahoe-dev@tahoe-lafs.org). For example: -https://tahoe-lafs.org/pipermail/tahoe-dev/2020-October/009995.html +mailing-list (tahoe-dev@lists.tahoe-lafs.org). For example: +https://lists.tahoe-lafs.org/pipermail/tahoe-dev/2020-October/009978.html Is The Release Done Yet? diff --git a/docs/running.rst b/docs/running.rst index a53f5d9e2..406c8200b 100644 --- a/docs/running.rst +++ b/docs/running.rst @@ -238,7 +238,7 @@ You can chat with other users of and hackers of this software on the #tahoe-lafs IRC channel at ``irc.libera.chat``, or on the `tahoe-dev mailing list`_. -.. _tahoe-dev mailing list: https://tahoe-lafs.org/cgi-bin/mailman/listinfo/tahoe-dev +.. _tahoe-dev mailing list: https://lists.tahoe-lafs.org/mailman/listinfo/tahoe-dev Complain diff --git a/misc/python3/depgraph.sh b/misc/python3/depgraph.sh index d5ad33bf7..4fd98a717 100755 --- a/misc/python3/depgraph.sh +++ b/misc/python3/depgraph.sh @@ -16,7 +16,7 @@ if git diff-index --quiet HEAD; then fi git config user.name 'Build Automation' -git config user.email 'tahoe-dev@tahoe-lafs.org' +git config user.email 'tahoe-dev@lists.tahoe-lafs.org' git add tahoe-deps.json tahoe-ported.json git commit -m "\ diff --git a/relnotes.txt b/relnotes.txt index d5552e24b..4afbd6cc5 100644 --- a/relnotes.txt +++ b/relnotes.txt @@ -155,7 +155,7 @@ Planet Earth [4] https://github.com/tahoe-lafs/tahoe-lafs/blob/tahoe-lafs-1.15.1/COPYING.GPL [5] https://github.com/tahoe-lafs/tahoe-lafs/blob/tahoe-lafs-1.15.1/COPYING.TGPPL.rst [6] https://tahoe-lafs.readthedocs.org/en/tahoe-lafs-1.15.1/INSTALL.html -[7] https://tahoe-lafs.org/cgi-bin/mailman/listinfo/tahoe-dev +[7] https://lists.tahoe-lafs.org/mailman/listinfo/tahoe-dev [8] https://tahoe-lafs.org/trac/tahoe-lafs/roadmap [9] https://github.com/tahoe-lafs/tahoe-lafs/blob/master/CREDITS [10] https://tahoe-lafs.org/trac/tahoe-lafs/wiki/Dev diff --git a/setup.py b/setup.py index 3433e93f4..e1d711ccf 100644 --- a/setup.py +++ b/setup.py @@ -359,7 +359,7 @@ setup(name="tahoe-lafs", # also set in __init__.py description='secure, decentralized, fault-tolerant file store', long_description=open('README.rst', 'r', encoding='utf-8').read(), author='the Tahoe-LAFS project', - author_email='tahoe-dev@tahoe-lafs.org', + author_email='tahoe-dev@lists.tahoe-lafs.org', url='https://tahoe-lafs.org/', license='GNU GPL', # see README.rst -- there is an alternative licence cmdclass={"update_version": UpdateVersion, From 7537a3a06ffed66b63700d0dec759a4cb3c810a5 Mon Sep 17 00:00:00 2001 From: Sajith Sasidharan Date: Tue, 31 Aug 2021 10:22:50 -0400 Subject: [PATCH 093/269] Hat tip to OSUOSL for hosting tahoe-dev list --- README.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.rst b/README.rst index e58211601..7608e8829 100644 --- a/README.rst +++ b/README.rst @@ -98,6 +98,8 @@ Before authoring or reviewing a patch, please familiarize yourself with the `Cod We would like to thank `Fosshost `__ for supporting us with hosting services. If your open source project needs help, you can apply for their support. +We are grateful to `Oregon State University Open Source Lab `__ for hosting tahoe-dev mailing list. + ❓ FAQ ------ From d9561070ae6f7227dce4706f0ae02a77e963fb47 Mon Sep 17 00:00:00 2001 From: fenn-cs Date: Wed, 1 Sep 2021 23:07:19 +0100 Subject: [PATCH 094/269] Python3 support note in Readme Signed-off-by: fenn-cs --- README.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.rst b/README.rst index 2cc6e38eb..ca491b353 100644 --- a/README.rst +++ b/README.rst @@ -56,6 +56,11 @@ For more detailed instructions, read `docs/INSTALL.rst `__ . Once tahoe --version works, see `docs/running.rst `__ to learn how to set up your first Tahoe-LAFS node. +🐍 `Python3` Support +-------------------- + +`Python 3` support has been introduced as from ` Tahoe-LAFS 1.16.x` alongside `Python 2`. System admnistrators are advised to start runing Tahoe on `Python 3` and should expect the drop of `Python 2` support any future version from now. Please, feel free to file issues if you run into bugs running Tahoe on `Python 3`. + 🤖 Issues --------- From 07ac248f3b462d4eb57ece4a536267db19f9820a Mon Sep 17 00:00:00 2001 From: fenn-cs Date: Wed, 1 Sep 2021 23:11:43 +0100 Subject: [PATCH 095/269] Python3 support note update Signed-off-by: fenn-cs --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index ca491b353..bb8c5c252 100644 --- a/README.rst +++ b/README.rst @@ -56,10 +56,10 @@ For more detailed instructions, read `docs/INSTALL.rst `__ . Once tahoe --version works, see `docs/running.rst `__ to learn how to set up your first Tahoe-LAFS node. -🐍 `Python3` Support +🐍 `Python 3` Support -------------------- -`Python 3` support has been introduced as from ` Tahoe-LAFS 1.16.x` alongside `Python 2`. System admnistrators are advised to start runing Tahoe on `Python 3` and should expect the drop of `Python 2` support any future version from now. Please, feel free to file issues if you run into bugs running Tahoe on `Python 3`. +`Python 3` support has been introduced as from `Tahoe-LAFS 1.16.x` alongside `Python 2`. System admnistrators are advised to start running Tahoe on `Python 3` and should expect the drop of `Python 2` support any future version from now. Please, feel free to file issues if you run into bugs while running Tahoe on `Python 3`. 🤖 Issues From 6c679bd4e0292af572026353d0e7a1c74e01184f Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 2 Sep 2021 11:35:39 -0400 Subject: [PATCH 096/269] Stop using callRemoteOnly. --- src/allmydata/immutable/layout.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/allmydata/immutable/layout.py b/src/allmydata/immutable/layout.py index 3db9f096e..886a5db73 100644 --- a/src/allmydata/immutable/layout.py +++ b/src/allmydata/immutable/layout.py @@ -15,7 +15,7 @@ from zope.interface import implementer from twisted.internet import defer from allmydata.interfaces import IStorageBucketWriter, IStorageBucketReader, \ FileTooLargeError, HASH_SIZE -from allmydata.util import mathutil, observer, pipeline +from allmydata.util import mathutil, observer, pipeline, log from allmydata.util.assertutil import precondition from allmydata.storage.server import si_b2a @@ -254,8 +254,7 @@ class WriteBucketProxy(object): return d def abort(self): - return self._rref.callRemoteOnly("abort") - + self._rref.callRemote("abort").addErrback(log.err) def get_servername(self): return self._server.get_name() From 16af282c193430c38287f475c8a85b3830c90f1a Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 2 Sep 2021 13:11:51 -0400 Subject: [PATCH 097/269] Clarify chunking; lift ordering requirement; document response status --- docs/proposed/http-storage-node-protocol.rst | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index dd696c340..5012d4143 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -458,17 +458,21 @@ The response includes ``already-have`` and ``allocated`` for two reasons: Write data for the indicated share. The share number must belong to the storage index. The request body is the raw share data (i.e., ``application/octet-stream``). -*Content-Range* requests are encouraged for large transfers. +*Content-Range* requests are encouraged for large transfers to allow partially complete uploads to be resumed. For example, -for a 1MiB share the data can be broken in to 8 128KiB chunks. -Each chunk can be *PUT* separately with the appropriate *Content-Range* header. +a 1MiB share can be divided in to eight separate 128KiB chunks. +Each chunk can be uploaded in a separate request. +Each request can include a *Content-Range* value indicating its placement within the complete share. +If any one of these requests fails then at most 128KiB of upload work needs to be retried. + The server must recognize when all of the data has been received and mark the share as complete (which it can do because it was informed of the size when the storage index was initialized). Clients should upload chunks in re-assembly order. -Servers may reject out-of-order chunks for implementation simplicity. -If an individual *PUT* fails then only a limited amount of effort is wasted on the necessary retry. -.. think about copying https://developers.google.com/drive/api/v2/resumable-upload + +When a chunk that does not complete the share is successfully uploaded the response is ``OK``. +When the chunk that completes the share is successfully uploaded the response is ``CREATED``. +If the *Content-Range* for a request covers part of the share that has already been uploaded the response is ``CONFLICT``. ``POST /v1/immutable/:storage_index/:share_number/corrupt`` !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! From 61b5c8873834d458e58bdbab7ee0fb7fb385211b Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 2 Sep 2021 13:16:20 -0400 Subject: [PATCH 098/269] reveal to clients what data is still required This lets a client recover from an upload that completes but for which the response is lost (eg because network error or client restart) --- docs/proposed/http-storage-node-protocol.rst | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index 5012d4143..d7a41e827 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -469,10 +469,21 @@ The server must recognize when all of the data has been received and mark the sh (which it can do because it was informed of the size when the storage index was initialized). Clients should upload chunks in re-assembly order. +* When a chunk that does not complete the share is successfully uploaded the response is ``OK``. +* When the chunk that completes the share is successfully uploaded the response is ``CREATED``. +* If the *Content-Range* for a request covers part of the share that has already been uploaded the response is ``CONFLICT``. + The response body indicates the range of share data that has yet to be uploaded. + That is:: + + { "required": + [ { "begin": + , "end": + } + , + ... + ] + } -When a chunk that does not complete the share is successfully uploaded the response is ``OK``. -When the chunk that completes the share is successfully uploaded the response is ``CREATED``. -If the *Content-Range* for a request covers part of the share that has already been uploaded the response is ``CONFLICT``. ``POST /v1/immutable/:storage_index/:share_number/corrupt`` !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! From ae9ec48e131b90dfc5118551cf7e8888bd650b32 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 2 Sep 2021 13:19:49 -0400 Subject: [PATCH 099/269] news fragment --- newsfragments/3769.documentation | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/3769.documentation diff --git a/newsfragments/3769.documentation b/newsfragments/3769.documentation new file mode 100644 index 000000000..3d4ef7d4c --- /dev/null +++ b/newsfragments/3769.documentation @@ -0,0 +1 @@ +The Great Black Swamp specification now allows parallel upload of immutable share data. From 63bfff19e9be9665461b4355e56ced50a227d2a8 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 2 Sep 2021 15:05:15 -0400 Subject: [PATCH 100/269] Don't rely on Foolscap's semantics. --- src/allmydata/mutable/servermap.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/src/allmydata/mutable/servermap.py b/src/allmydata/mutable/servermap.py index 4f3226649..211b1fc16 100644 --- a/src/allmydata/mutable/servermap.py +++ b/src/allmydata/mutable/servermap.py @@ -607,13 +607,14 @@ class ServermapUpdater(object): return d def _do_read(self, server, storage_index, shnums, readv): + """ + If self._add_lease is true, a lease is added, and the result only fires + once the least has also been added. + """ ss = server.get_storage_server() if self._add_lease: # send an add-lease message in parallel. The results are handled - # separately. This is sent before the slot_readv() so that we can - # be sure the add_lease is retired by the time slot_readv comes - # back (this relies upon our knowledge that the server code for - # add_lease is synchronous). + # separately. renew_secret = self._node.get_renewal_secret(server) cancel_secret = self._node.get_cancel_secret(server) d2 = ss.add_lease( @@ -623,7 +624,16 @@ class ServermapUpdater(object): ) # we ignore success d2.addErrback(self._add_lease_failed, server, storage_index) + else: + d2 = defer.succeed(None) d = ss.slot_readv(storage_index, shnums, readv) + + def passthrough(result): + # Wait for d2, but fire with result of slot_readv() regardless of + # result of d2. + return d2.addBoth(lambda _: result) + + d.addCallback(passthrough) return d From 789a7edb56501e5e7d95c93e7570bb4fc1197507 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 2 Sep 2021 15:21:42 -0400 Subject: [PATCH 101/269] Get rid of more callRemoteOnly usage. --- src/allmydata/immutable/downloader/share.py | 4 +++- src/allmydata/storage_client.py | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/allmydata/immutable/downloader/share.py b/src/allmydata/immutable/downloader/share.py index 86b99e99c..1e751500b 100644 --- a/src/allmydata/immutable/downloader/share.py +++ b/src/allmydata/immutable/downloader/share.py @@ -475,7 +475,9 @@ class Share(object): # there was corruption somewhere in the given range reason = "corruption in share[%d-%d): %s" % (start, start+offset, str(f.value)) - self._rref.callRemoteOnly("advise_corrupt_share", reason.encode("utf-8")) + self._rref.callRemote( + "advise_corrupt_share", reason.encode("utf-8") + ).addErrback(log.err) def _satisfy_block_hash_tree(self, needed_hashes): o_bh = self.actual_offsets["block_hashes"] diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 22dd09bcb..c278abce3 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -1009,10 +1009,10 @@ class _StorageServer(object): shnum, reason, ): - return self._rref.callRemoteOnly( + self._rref.callRemote( "advise_corrupt_share", share_type, storage_index, shnum, reason, - ) + ).addErrback(log.err) From 04d2b27f46d9dee1122947e1f5bf50ac87a5b4aa Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 2 Sep 2021 15:24:19 -0400 Subject: [PATCH 102/269] News file. --- newsfragments/3779.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3779.minor diff --git a/newsfragments/3779.minor b/newsfragments/3779.minor new file mode 100644 index 000000000..e69de29bb From 78f70d6bdc1853537d7cc865f69368be541f746d Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 2 Sep 2021 15:53:14 -0400 Subject: [PATCH 103/269] write some words about lease renewal secrets --- docs/proposed/http-storage-node-protocol.rst | 5 +++ docs/specifications/index.rst | 1 + docs/specifications/lease.rst | 41 ++++++++++++++++++++ 3 files changed, 47 insertions(+) create mode 100644 docs/specifications/lease.rst diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index d7a41e827..09c193c65 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -380,6 +380,11 @@ then the expiration time of that lease will be changed to 31 days after the time If it does not match an existing lease then a new lease will be created with this ``renew-secret`` which expires 31 days after the time of this operation. +The renew and cancel secrets must be 32 bytes long +(or in the case of JSON encoding they must UTF-8 encode to 32 bytes). +The server treats them as opaque values. +See `leases`_ for details about how the Tahoe-LAFS storage client constructs these values. + In these cases the response is ``NO CONTENT`` with an empty body. It is possible that the storage server will have no shares for the given ``storage_index`` because: diff --git a/docs/specifications/index.rst b/docs/specifications/index.rst index 2029c9e5a..e813acf07 100644 --- a/docs/specifications/index.rst +++ b/docs/specifications/index.rst @@ -14,5 +14,6 @@ the data formats used by Tahoe. URI-extension mutable dirnodes + lease servers-of-happiness backends/raic diff --git a/docs/specifications/lease.rst b/docs/specifications/lease.rst new file mode 100644 index 000000000..fd5535701 --- /dev/null +++ b/docs/specifications/lease.rst @@ -0,0 +1,41 @@ +.. -*- coding: utf-8 -*- + +Share Leases +============ + +A lease is a marker attached to a share indicating that some client has asked for that share to be retained for some amount of time. +The intent is to allow clients and servers to collaborate to determine which data should still be retained and which can be discarded to reclaim storage space. +Zero or more leases may be attached to any particular share. + +Renewal Secrets +--------------- + +Each lease is uniquely identified by its **renewal secret**. +This is a 32 byte string which can be used to extend the validity period of that lease. + +To a storage server a renewal secret is an opaque value which is only ever compared to other renewal secrets to determine equality. + +Storage clients will typically want to follow a scheme to deterministically derive the renewal secret for a particular share from information the client already holds about that share. +This allows a client to maintain and renew single long-lived lease without maintaining additional local state. + +The scheme in use in Tahoe-LAFS as of 1.16.0 is as follows. + +* The **netstring encoding** of a byte string is the concatenation of: + * the base 10 representation of the length of the string + * ":" + * the string + * "," +* The **sha256d digest** is the **sha256 digest** of the **sha256 digest** of a string. +* The **sha256d tagged digest** is the **sha256d digest** of the concatenation of the **netstring encoding** of one string with one other unmodified string. +* The **sha256d tagged pair digest** the **sha256d digest** of the concatenation of the **netstring encodings** of each of three strings. +* The **bucket renewal tag** is ``allmydata_bucket_renewal_secret_v1``. +* The **file renewal tag** is ``allmydata_file_renewal_secret_v1``. +* The **client renewal tag** is ``allmydata_client_renewal_secret_v1``. +* The **lease secret** is a 32 byte string, typically randomly generated once and then persisted for all future uses. +* The **client renewal secret** is the **sha256d tagged digest** of (**lease secret**, **client renewal tag**). +* The **storage index** is constructed using a capability-type-specific scheme. + See ``storage_index_hash`` and ``ssk_storage_index_hash`` calls in ``src/allmydata/uri.py``. +* The **file renewal secret** is the **sha256d tagged pair digest** of (**file renewal tag**, **client renewal secret**, **storage index**). +* The **base32 encoding** is ``base64.b32encode`` lowercased and with trailing ``=`` stripped. +* The **peer id** is the **base32 encoding** of the SHA1 digest of the server's x509 certificate. +* The **renewal secret** is the **sha256d tagged pair digest** of (**bucket renewal tag**, **file renewal secret**, **peer id**). From b6173eea839a21cdefedb6117cb5edf9b0fbee02 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 2 Sep 2021 16:42:27 -0400 Subject: [PATCH 104/269] news fragment --- docs/3774.documentation | 1 + 1 file changed, 1 insertion(+) create mode 100644 docs/3774.documentation diff --git a/docs/3774.documentation b/docs/3774.documentation new file mode 100644 index 000000000..d58105966 --- /dev/null +++ b/docs/3774.documentation @@ -0,0 +1 @@ +There is now a specification for the scheme which Tahoe-LAFS storage clients use to derive their lease renewal secrets. From 11a0b8d20979b1d0340819bd94b6b614496abde0 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 2 Sep 2021 16:44:42 -0400 Subject: [PATCH 105/269] attempt to appease rst --- docs/specifications/lease.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/specifications/lease.rst b/docs/specifications/lease.rst index fd5535701..bf8e8f181 100644 --- a/docs/specifications/lease.rst +++ b/docs/specifications/lease.rst @@ -21,10 +21,12 @@ This allows a client to maintain and renew single long-lived lease without maint The scheme in use in Tahoe-LAFS as of 1.16.0 is as follows. * The **netstring encoding** of a byte string is the concatenation of: + * the base 10 representation of the length of the string * ":" * the string * "," + * The **sha256d digest** is the **sha256 digest** of the **sha256 digest** of a string. * The **sha256d tagged digest** is the **sha256d digest** of the concatenation of the **netstring encoding** of one string with one other unmodified string. * The **sha256d tagged pair digest** the **sha256d digest** of the concatenation of the **netstring encodings** of each of three strings. From bb63331720f3e3d32fc6e2775e057064a1cc28b7 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 3 Sep 2021 09:04:08 -0400 Subject: [PATCH 106/269] put the newsfragment in the right place --- {docs => newsfragments}/3774.documentation | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename {docs => newsfragments}/3774.documentation (100%) diff --git a/docs/3774.documentation b/newsfragments/3774.documentation similarity index 100% rename from docs/3774.documentation rename to newsfragments/3774.documentation From 3ba379ce7e90736adbc2b325f9965117bec09690 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 3 Sep 2021 09:06:27 -0400 Subject: [PATCH 107/269] some formatting improvements --- docs/specifications/lease.rst | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/specifications/lease.rst b/docs/specifications/lease.rst index bf8e8f181..a8bee427e 100644 --- a/docs/specifications/lease.rst +++ b/docs/specifications/lease.rst @@ -22,17 +22,17 @@ The scheme in use in Tahoe-LAFS as of 1.16.0 is as follows. * The **netstring encoding** of a byte string is the concatenation of: - * the base 10 representation of the length of the string - * ":" - * the string - * "," + * the ascii encoding of the base 10 representation of the length of the string + * ``":"`` + * the string itself + * ``","`` * The **sha256d digest** is the **sha256 digest** of the **sha256 digest** of a string. * The **sha256d tagged digest** is the **sha256d digest** of the concatenation of the **netstring encoding** of one string with one other unmodified string. * The **sha256d tagged pair digest** the **sha256d digest** of the concatenation of the **netstring encodings** of each of three strings. -* The **bucket renewal tag** is ``allmydata_bucket_renewal_secret_v1``. -* The **file renewal tag** is ``allmydata_file_renewal_secret_v1``. -* The **client renewal tag** is ``allmydata_client_renewal_secret_v1``. +* The **bucket renewal tag** is ``"allmydata_bucket_renewal_secret_v1"``. +* The **file renewal tag** is ``"allmydata_file_renewal_secret_v1"``. +* The **client renewal tag** is ``"allmydata_client_renewal_secret_v1"``. * The **lease secret** is a 32 byte string, typically randomly generated once and then persisted for all future uses. * The **client renewal secret** is the **sha256d tagged digest** of (**lease secret**, **client renewal tag**). * The **storage index** is constructed using a capability-type-specific scheme. From 8fe9532faf64c9d65f46f0ecf92d38af2634568e Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 3 Sep 2021 09:17:34 -0400 Subject: [PATCH 108/269] get the cross-reference right --- docs/proposed/http-storage-node-protocol.rst | 2 +- docs/specifications/lease.rst | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index 09c193c65..47f94db49 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -383,7 +383,7 @@ then a new lease will be created with this ``renew-secret`` which expires 31 day The renew and cancel secrets must be 32 bytes long (or in the case of JSON encoding they must UTF-8 encode to 32 bytes). The server treats them as opaque values. -See `leases`_ for details about how the Tahoe-LAFS storage client constructs these values. +:ref:`Share Leases` gives details about how the Tahoe-LAFS storage client constructs these values. In these cases the response is ``NO CONTENT`` with an empty body. diff --git a/docs/specifications/lease.rst b/docs/specifications/lease.rst index a8bee427e..54ee5cd28 100644 --- a/docs/specifications/lease.rst +++ b/docs/specifications/lease.rst @@ -1,5 +1,7 @@ .. -*- coding: utf-8 -*- +.. _share leases: + Share Leases ============ From 4000116c244aaa6dda0c4cb87f7fc9bf7d332cb8 Mon Sep 17 00:00:00 2001 From: Florian Sesser Date: Fri, 3 Sep 2021 13:42:09 +0000 Subject: [PATCH 109/269] Newsfragment for OpenMetrics endpoint --- newsfragments/3786.feature | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/3786.feature diff --git a/newsfragments/3786.feature b/newsfragments/3786.feature new file mode 100644 index 000000000..82ce3f974 --- /dev/null +++ b/newsfragments/3786.feature @@ -0,0 +1 @@ +tahoe-lafs now provides its statistics also in OpenMetrics format (for Prometheus et. al.) on `/statistics?t=openmetrics`. From 8a64f50b79cefb8d76ce6111b51290c255099caf Mon Sep 17 00:00:00 2001 From: Florian Sesser Date: Fri, 3 Sep 2021 14:40:34 +0000 Subject: [PATCH 110/269] WIP - Could be wronger --- src/allmydata/web/status.py | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/src/allmydata/web/status.py b/src/allmydata/web/status.py index 8c78b1156..741f93061 100644 --- a/src/allmydata/web/status.py +++ b/src/allmydata/web/status.py @@ -14,6 +14,7 @@ from past.builtins import long import itertools import hashlib +import re from twisted.internet import defer from twisted.python.filepath import FilePath from twisted.web.resource import Resource @@ -1553,18 +1554,22 @@ class Statistics(MultiFormatResource): @render_exception def render_OPENMETRICS(self, req): - req.setHeader("content-type", "application/openmetrics-text") - return "3. occurence. This should be the one.\n" + def mangle_name(name): + return re.sub( + "_(\d\d)_(\d)_percentile", + '{quantile="0.\g<1>\g<2>"}', + name.replace(".", "_") + ) - # @render_exception - # def render_OPENMETRICS(self, req): - # req.setHeader("content-type", "text/plain") - # if self._helper: - # stats = self._helper.get_stats() - # import pprint - # return pprint.PrettyPrinter().pprint(stats) + "\n" - # return "uh oh\n" + req.setHeader( + "content-type", "application/openmetrics-text; version=1.0.0; charset=utf-8" + ) + stats = self._provider.get_stats() + return (str({mangle_name(k): v for k, v in stats['counters'].items()}) + + str({mangle_name(k): v for k, v in stats['stats'].items()}) + + "\n" + ) class StatisticsElement(Element): From 78a1d65b784b50486c8c59f8cdc5bad706405797 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 3 Sep 2021 12:33:07 -0400 Subject: [PATCH 111/269] RFC 7049, section 4.1 describes correct JSON encoding for byte strings --- docs/proposed/http-storage-node-protocol.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index 47f94db49..bc44468a6 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -380,8 +380,7 @@ then the expiration time of that lease will be changed to 31 days after the time If it does not match an existing lease then a new lease will be created with this ``renew-secret`` which expires 31 days after the time of this operation. -The renew and cancel secrets must be 32 bytes long -(or in the case of JSON encoding they must UTF-8 encode to 32 bytes). +The renew and cancel secrets must be 32 bytes long. The server treats them as opaque values. :ref:`Share Leases` gives details about how the Tahoe-LAFS storage client constructs these values. From a864bd51323f4e1378ced63a50516bad645e559f Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 3 Sep 2021 12:44:23 -0400 Subject: [PATCH 112/269] more precision --- docs/proposed/http-storage-node-protocol.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index bc44468a6..ade0bc167 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -380,7 +380,7 @@ then the expiration time of that lease will be changed to 31 days after the time If it does not match an existing lease then a new lease will be created with this ``renew-secret`` which expires 31 days after the time of this operation. -The renew and cancel secrets must be 32 bytes long. +``renew-secret`` and ``cancel-secret`` values must be 32 bytes long. The server treats them as opaque values. :ref:`Share Leases` gives details about how the Tahoe-LAFS storage client constructs these values. From bb57fcfb50d4e01bbc4de2e23dbbf7a60c004031 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 3 Sep 2021 12:45:45 -0400 Subject: [PATCH 113/269] words about the cancel secret --- docs/specifications/lease.rst | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/docs/specifications/lease.rst b/docs/specifications/lease.rst index 54ee5cd28..8fc7c42eb 100644 --- a/docs/specifications/lease.rst +++ b/docs/specifications/lease.rst @@ -43,3 +43,21 @@ The scheme in use in Tahoe-LAFS as of 1.16.0 is as follows. * The **base32 encoding** is ``base64.b32encode`` lowercased and with trailing ``=`` stripped. * The **peer id** is the **base32 encoding** of the SHA1 digest of the server's x509 certificate. * The **renewal secret** is the **sha256d tagged pair digest** of (**bucket renewal tag**, **file renewal secret**, **peer id**). + +Cancel Secrets +-------------- + +Lease cancellation is unimplemented. +Nevertheless, +a cancel secret is sent by storage clients to storage servers and stored in lease records. + +The scheme for deriving **cancel secret** in use in Tahoe-LAFS as of 1.16.0 is similar to that used to derive the **renewal secret**. + +The differences are: + +* Use of **client renewal tag** is replaced by use of **client cancel tag**. +* Use of **file renewal secret** is replaced by use of **file cancel tag**. +* Use of **bucket renewal tag** is replaced by use of **bucket cancel tag**. +* **client cancel tag** is ``"allmydata_client_cancel_secret_v1"``. +* **file cancel tag** is ``"allmydata_file_cancel_secret_v1"``. +* **bucket cancel tag** is ``"allmydata_bucket_cancel_secret_v1"``. From 148a0573dea02d67e9b05e1156afe47807402a96 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 3 Sep 2021 13:11:02 -0400 Subject: [PATCH 114/269] Replace colon on filename only, not on whole path. This would break Windows logging of corruption reports, since colon would be removed from e.g. "C:". --- src/allmydata/storage/server.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py index 56b47ae89..f4996756e 100644 --- a/src/allmydata/storage/server.py +++ b/src/allmydata/storage/server.py @@ -708,8 +708,10 @@ class StorageServer(service.MultiService, Referenceable): now = time_format.iso_utc(sep="T") si_s = si_b2a(storage_index) # windows can't handle colons in the filename - fn = os.path.join(self.corruption_advisory_dir, - "%s--%s-%d" % (now, str(si_s, "utf-8"), shnum)).replace(":","") + fn = os.path.join( + self.corruption_advisory_dir, + ("%s--%s-%d" % (now, str(si_s, "utf-8"), shnum)).replace(":","") + ) with open(fn, "w") as f: f.write("report: Share Corruption\n") f.write("type: %s\n" % bytes_to_native_str(share_type)) From c7e82b1640238b9e8af65d3f05a7a21647cba183 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 3 Sep 2021 13:12:57 -0400 Subject: [PATCH 115/269] Guess it's a bugfix. --- newsfragments/3779.bugfix | 1 + newsfragments/3779.minor | 0 2 files changed, 1 insertion(+) create mode 100644 newsfragments/3779.bugfix delete mode 100644 newsfragments/3779.minor diff --git a/newsfragments/3779.bugfix b/newsfragments/3779.bugfix new file mode 100644 index 000000000..c6745f1af --- /dev/null +++ b/newsfragments/3779.bugfix @@ -0,0 +1 @@ +Fixed bug where share corruption events were not recorded on Windows. \ No newline at end of file diff --git a/newsfragments/3779.minor b/newsfragments/3779.minor deleted file mode 100644 index e69de29bb..000000000 From 51b1e5624af8553e16ab3b0686da261e4c2b5917 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 3 Sep 2021 14:04:10 -0400 Subject: [PATCH 116/269] Skeleton setting up the test infrastructure. --- src/allmydata/test/test_istorageserver.py | 38 +++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 src/allmydata/test/test_istorageserver.py diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py new file mode 100644 index 000000000..51d5da2c0 --- /dev/null +++ b/src/allmydata/test/test_istorageserver.py @@ -0,0 +1,38 @@ +""" +Tests for the ``IStorageServer`` interface. +""" + +from twisted.trial import unittest +from twisted.internet.defer import inlineCallbacks + +from allmydata.interfaces import IStorageServer +from .test_system import SystemTestMixin + + +class IStorageServerTestsMixin: + """ + Tests for ``IStorageServer``. + + ``self.storage_server`` is expected to provide ``IStorageServer``. + """ + @inlineCallbacks + def test_version(self): + yield self.storage_server.get_version() + + +class FoolscapIStorageServerTests( + SystemTestMixin, IStorageServerTestsMixin, unittest.TestCase +): + """Run tests on Foolscap version of ``IStorageServer.""" + + @inlineCallbacks + def setUp(self): + self.basedir = "test_istorageserver/{}/{}".format( + self.__class__.__name__, self._testMethodName + ) + yield SystemTestMixin.setUp(self) + yield self.set_up_nodes(1) + self.storage_server = next( + iter(self.clients[0].storage_broker.get_known_servers()) + ).get_storage_server() + self.assertTrue(IStorageServer.providedBy(self.storage_server)) From f0fe323fa1e32555aeb21d9332ad30ff319084b7 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 3 Sep 2021 15:14:09 -0400 Subject: [PATCH 117/269] Simplify the immutable share reading interface --- docs/proposed/http-storage-node-protocol.rst | 25 +++++++------------- 1 file changed, 9 insertions(+), 16 deletions(-) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index d7a41e827..fcf1e43fe 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -509,28 +509,21 @@ For example:: [1, 5] -``GET /v1/immutable/:storage_index?share=:s0&share=:sN&offset=o1&size=z0&offset=oN&size=zN`` -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +``GET /v1/immutable/:storage_index/:share_number?offset=:offset&length=:length +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -Read data from the indicated immutable shares. -If ``share`` query parameters are given, selecte only those shares for reading. -Otherwise, select all shares present. -If ``size`` and ``offset`` query parameters are given, -only the portions thus identified of the selected shares are returned. -Otherwise, all data is from the selected shares is returned. +Read a contiguous sequence of bytes from one share in one bucket. +If the ``offset`` query parameter is given then it is interpreted as a base 10 representation of an integer giving the position at which to begin reading. +If it is not given then begin reading at the beginning of the share. +If the ``length`` query parameter is given then it is interpreted as a base 10 representation of an integer giving the maximum number of bytes to read and return. +If it is not given then bytes will be read until the end of the share is reached. -The response body contains a mapping giving the read data. -For example:: - - { - 3: ["foo", "bar"], - 7: ["baz", "quux"] - } +The response body is the raw share data (i.e., ``application/octet-stream``). Discussion `````````` -Offset and size of the requested data are specified here as query arguments. +Offset and length of the requested data are specified here as query arguments. Instead, this information could be present in a ``Range`` header in the request. This is the more obvious choice and leverages an HTTP feature built for exactly this use-case. However, HTTP requires that the ``Content-Type`` of the response to "range requests" be ``multipart/...``. From 895c77fecf9559a31a7770ae98841846124e995e Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 3 Sep 2021 15:15:48 -0400 Subject: [PATCH 118/269] news fragment --- newsfragments/3777.documentation | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/3777.documentation diff --git a/newsfragments/3777.documentation b/newsfragments/3777.documentation new file mode 100644 index 000000000..7635cc1e6 --- /dev/null +++ b/newsfragments/3777.documentation @@ -0,0 +1 @@ +The Great Black Swamp proposed specification now has a simplified interface for reading data from immutable shares. From 18cbff1789c6f3525c3f8127b629f534bc5c9a9f Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 3 Sep 2021 15:22:29 -0400 Subject: [PATCH 119/269] minimal discussion about this change --- docs/proposed/http-storage-node-protocol.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index fcf1e43fe..fb327b0a8 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -534,6 +534,15 @@ There are many drawbacks to this framing technique: 2. It is resource-intensive to parse. 3. It is complex to parse safely [#]_ [#]_ [#]_ [#]_. +A previous revision of this specification allowed requesting one or more contiguous sequences from one or more shares. +This *superficially* mirrored the Foolscap based interface somewhat closely. +The interface was simplified to this version because this version is all that is required to let clients retrieve any desired information. +It only requires that the client issue multiple requests. +This can be done with pipelining or parallel requests to avoid an additional latency penalty. +In the future, +if there are performance goals, +benchmarks can demonstrate whether they are achieved by a more complicated interface or some other change. + Mutable ------- From 8d15a0d5ebcbf16979a984484807a0c54f6b4a24 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 3 Sep 2021 16:27:57 -0400 Subject: [PATCH 120/269] words about authorization --- docs/proposed/http-storage-node-protocol.rst | 26 ++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index d7a41e827..0daa48cab 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -24,11 +24,21 @@ Glossary storage server a Tahoe-LAFS process configured to offer storage and reachable over the network for store and retrieve operations + storage service + a Python object held in memory in the storage server which provides the implementation of the storage protocol + introducer a Tahoe-LAFS process at a known location configured to re-publish announcements about the location of storage servers fURL a self-authenticating URL-like string which can be used to locate a remote object using the Foolscap protocol + (the storage service is an example of such an object) + + NURL + a self-authenticating URL-like string almost exactly like a fURL but without being tied to Foolscap + + swissnum + a short random string which is part of a fURL and which acts as a shared secret to authorize clients to use a storage service lease state associated with a share informing a storage server of the duration of storage desired by a client @@ -128,6 +138,8 @@ The Foolscap-based protocol offers: * A careful configuration of the TLS connection parameters *may* also offer **forward secrecy**. However, Tahoe-LAFS' use of Foolscap takes no steps to ensure this is the case. +* **Storage authorization** by way of a capability contained in the fURL addressing a storage service. + Discussion !!!!!!!!!! @@ -158,6 +170,10 @@ there is no way to write data which appears legitimate to a legitimate client). Therefore, **message confidentiality** is necessary when exchanging these secrets. **Forward secrecy** is preferred so that an attacker recording an exchange today cannot launch this attack at some future point after compromising the necessary keys. +A storage service offers service only to some clients. +A client proves their authorization to use the storage service by presenting a shared secret taken from the fURL. +In this way **storage authorization** is performed to prevent disallowed parties from consuming all available storage resources. + Functionality ------------- @@ -214,6 +230,10 @@ Additionally, by continuing to interact using TLS, Bob's client and Alice's storage node are assured of both **message authentication** and **message confidentiality**. +Bob's client further inspects the fURL for the *swissnum*. +When Bob's client issues HTTP requests to Alice's storage node it includes the *swissnum* in its requests. +**Storage authorization** has been achieved. + .. note:: Foolscap TubIDs are 20 bytes (SHA1 digest of the certificate). @@ -343,6 +363,12 @@ one branch contains all of the share data; another branch contains all of the lease data; etc. +Authorization is required for all endpoints. +The standard HTTP authorization protocol is used. +The authentication *type* used is ``Tahoe-LAFS``. +The swissnum from the NURL used to locate the storage service is used as the *credentials*. +If credentials are not presented or the swissnum is not associated with a storage service then no storage processing is performed and the request receives an ``UNAUTHORIZED`` response. + General ~~~~~~~ From a4334b35a0bb6f669429306187cc0fa93cef2518 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 3 Sep 2021 16:28:27 -0400 Subject: [PATCH 121/269] news fragment --- newsfragments/3785.documentation | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/3785.documentation diff --git a/newsfragments/3785.documentation b/newsfragments/3785.documentation new file mode 100644 index 000000000..4eb268f79 --- /dev/null +++ b/newsfragments/3785.documentation @@ -0,0 +1 @@ +The Great Black Swamp specification now describes the required authorization scheme. From e2b483e093b11cd31a513b2069755df7a0bf13ed Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 7 Sep 2021 08:13:03 -0400 Subject: [PATCH 122/269] an even stronger prevention is provided --- docs/proposed/http-storage-node-protocol.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index 0daa48cab..0c6186bc1 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -172,7 +172,7 @@ Therefore, **message confidentiality** is necessary when exchanging these secret A storage service offers service only to some clients. A client proves their authorization to use the storage service by presenting a shared secret taken from the fURL. -In this way **storage authorization** is performed to prevent disallowed parties from consuming all available storage resources. +In this way **storage authorization** is performed to prevent disallowed parties from consuming any storage resources. Functionality ------------- From 7219291343299263050ddaf7e39ac1f6af2c62b7 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 7 Sep 2021 13:30:21 -0400 Subject: [PATCH 123/269] add a reference implementation for lease renewal secret derivation --- docs/proposed/http-storage-node-protocol.rst | 2 +- docs/specifications/derive_renewal_secret.py | 87 ++++++++++++++++++++ 2 files changed, 88 insertions(+), 1 deletion(-) create mode 100644 docs/specifications/derive_renewal_secret.py diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index ade0bc167..3a09ccae0 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -45,7 +45,7 @@ Glossary (sometimes "slot" is considered a synonym for "storage index of a slot") storage index - a short string which can address a slot or a bucket + a 16 byte string which can address a slot or a bucket (in practice, derived by hashing the encryption key associated with contents of that slot or bucket) write enabler diff --git a/docs/specifications/derive_renewal_secret.py b/docs/specifications/derive_renewal_secret.py new file mode 100644 index 000000000..75009eda4 --- /dev/null +++ b/docs/specifications/derive_renewal_secret.py @@ -0,0 +1,87 @@ + +""" +This is a reference implementation of the lease renewal secret derivation +protocol in use by Tahoe-LAFS clients as of 1.16.0. +""" + +from allmydata.util.base32 import ( + a2b as b32decode, + b2a as b32encode, +) +from allmydata.util.hashutil import ( + tagged_hash, + tagged_pair_hash, +) + + +def derive_renewal_secret(lease_secret: bytes, storage_index: bytes, tubid: bytes) -> bytes: + assert len(lease_secret) == 32 + assert len(storage_index) == 16 + assert len(tubid) == 20 + + bucket_renewal_tag = b"allmydata_bucket_renewal_secret_v1" + file_renewal_tag = b"allmydata_file_renewal_secret_v1" + client_renewal_tag = b"allmydata_client_renewal_secret_v1" + + client_renewal_secret = tagged_hash(lease_secret, client_renewal_tag) + file_renewal_secret = tagged_pair_hash( + file_renewal_tag, + client_renewal_secret, + storage_index, + ) + peer_id = tubid + + return tagged_pair_hash(bucket_renewal_tag, file_renewal_secret, peer_id) + +def demo(): + secret = b32encode(derive_renewal_secret( + b"lease secretxxxxxxxxxxxxxxxxxxxx", + b"storage indexxxx", + b"tub idxxxxxxxxxxxxxx", + )).decode("ascii") + print("An example renewal secret: {}".format(secret)) + +def test(): + # These test vectors created by intrumenting Tahoe-LAFS + # bb57fcfb50d4e01bbc4de2e23dbbf7a60c004031 to emit `self.renew_secret` in + # allmydata.immutable.upload.ServerTracker.query and then uploading a + # couple files to a couple different storage servers. + test_vector = [ + dict(lease_secret=b"boity2cdh7jvl3ltaeebuiobbspjmbuopnwbde2yeh4k6x7jioga", + storage_index=b"vrttmwlicrzbt7gh5qsooogr7u", + tubid=b"v67jiisoty6ooyxlql5fuucitqiok2ic", + expected=b"osd6wmc5vz4g3ukg64sitmzlfiaaordutrez7oxdp5kkze7zp5zq", + ), + dict(lease_secret=b"boity2cdh7jvl3ltaeebuiobbspjmbuopnwbde2yeh4k6x7jioga", + storage_index=b"75gmmfts772ww4beiewc234o5e", + tubid=b"v67jiisoty6ooyxlql5fuucitqiok2ic", + expected=b"35itmusj7qm2pfimh62snbyxp3imreofhx4djr7i2fweta75szda", + ), + dict(lease_secret=b"boity2cdh7jvl3ltaeebuiobbspjmbuopnwbde2yeh4k6x7jioga", + storage_index=b"75gmmfts772ww4beiewc234o5e", + tubid=b"lh5fhobkjrmkqjmkxhy3yaonoociggpz", + expected=b"srrlruge47ws3lm53vgdxprgqb6bz7cdblnuovdgtfkqrygrjm4q", + ), + dict(lease_secret=b"vacviff4xfqxsbp64tdr3frg3xnkcsuwt5jpyat2qxcm44bwu75a", + storage_index=b"75gmmfts772ww4beiewc234o5e", + tubid=b"lh5fhobkjrmkqjmkxhy3yaonoociggpz", + expected=b"b4jledjiqjqekbm2erekzqumqzblegxi23i5ojva7g7xmqqnl5pq", + ), + ] + + for n, item in enumerate(test_vector): + derived = b32encode(derive_renewal_secret( + b32decode(item["lease_secret"]), + b32decode(item["storage_index"]), + b32decode(item["tubid"]), + )) + assert derived == item["expected"] , \ + "Test vector {} failed: {} (expected) != {} (derived)".format( + n, + item["expected"], + derived, + ) + print("{} test vectors validated".format(len(test_vector))) + +test() +demo() From ee224305b7a64f071103d387a7eb646f197710b7 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 7 Sep 2021 13:37:12 -0400 Subject: [PATCH 124/269] link to the reference implementation --- docs/specifications/lease.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/specifications/lease.rst b/docs/specifications/lease.rst index 8fc7c42eb..16adef0a7 100644 --- a/docs/specifications/lease.rst +++ b/docs/specifications/lease.rst @@ -44,6 +44,12 @@ The scheme in use in Tahoe-LAFS as of 1.16.0 is as follows. * The **peer id** is the **base32 encoding** of the SHA1 digest of the server's x509 certificate. * The **renewal secret** is the **sha256d tagged pair digest** of (**bucket renewal tag**, **file renewal secret**, **peer id**). +A reference implementation is available. + +.. literalinclude:: derive_renewal_secret.py + :language: python + :linenos: + Cancel Secrets -------------- From 82f94ae5af4791ff8d322f908432ae8c1fc7b2d0 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 7 Sep 2021 14:17:38 -0400 Subject: [PATCH 125/269] Yay use the Range request header --- docs/proposed/http-storage-node-protocol.rst | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index eb5154e45..5f67fcaa6 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -539,24 +539,20 @@ For example:: [1, 5] -``GET /v1/immutable/:storage_index/:share_number?offset=:offset&length=:length -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +``GET /v1/immutable/:storage_index/:share_number +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! Read a contiguous sequence of bytes from one share in one bucket. -If the ``offset`` query parameter is given then it is interpreted as a base 10 representation of an integer giving the position at which to begin reading. -If it is not given then begin reading at the beginning of the share. -If the ``length`` query parameter is given then it is interpreted as a base 10 representation of an integer giving the maximum number of bytes to read and return. -If it is not given then bytes will be read until the end of the share is reached. - The response body is the raw share data (i.e., ``application/octet-stream``). +The ``Range`` header may be used to request exactly one ``bytes`` range. +Interpretation and response behavior is as specified in RFC 7233 § 4.1. +Multiple ranges in a single request are *not* supported. Discussion `````````` -Offset and length of the requested data are specified here as query arguments. -Instead, this information could be present in a ``Range`` header in the request. -This is the more obvious choice and leverages an HTTP feature built for exactly this use-case. -However, HTTP requires that the ``Content-Type`` of the response to "range requests" be ``multipart/...``. +Multiple ``bytes`` ranges are not supported. +HTTP requires that the ``Content-Type`` of the response in that case be ``multipart/...``. The ``multipart`` major type brings along string sentinel delimiting as a means to frame the different response parts. There are many drawbacks to this framing technique: From 5a238c4f072749acc0b1ff2b2ec798ab180688f4 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 7 Sep 2021 15:12:24 -0400 Subject: [PATCH 126/269] Nope, that doesn't sound like an enhancement at all. --- src/allmydata/test/cli/test_status.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/allmydata/test/cli/test_status.py b/src/allmydata/test/cli/test_status.py index 0d9e23c83..a015391e2 100644 --- a/src/allmydata/test/cli/test_status.py +++ b/src/allmydata/test/cli/test_status.py @@ -139,9 +139,6 @@ class CommandStatus(unittest.TestCase): """ These tests just exercise the renderers and ensure they don't catastrophically fail. - - They could be enhanced to look for "some" magic strings in the - results and assert they're in the output. """ def setUp(self): From 4db8c2209b47a2917f14f868f9f895ad34d12c27 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 7 Sep 2021 15:16:30 -0400 Subject: [PATCH 127/269] lint --- src/allmydata/scripts/tahoe_status.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/scripts/tahoe_status.py b/src/allmydata/scripts/tahoe_status.py index 5d61fcf5b..13677b189 100644 --- a/src/allmydata/scripts/tahoe_status.py +++ b/src/allmydata/scripts/tahoe_status.py @@ -11,7 +11,7 @@ if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import os -from urllib.parse import urlencode, quote as url_quote +from urllib.parse import urlencode import json From 24e7dcfcd882f55d08406d9fea7df5a35a3d9905 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 7 Sep 2021 16:10:10 -0400 Subject: [PATCH 128/269] Let me write unicode to stdout on Python 2 --- src/allmydata/scripts/tahoe_status.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/allmydata/scripts/tahoe_status.py b/src/allmydata/scripts/tahoe_status.py index 13677b189..9b2c89d24 100644 --- a/src/allmydata/scripts/tahoe_status.py +++ b/src/allmydata/scripts/tahoe_status.py @@ -21,6 +21,26 @@ from allmydata.scripts.common_http import BadResponse from allmydata.util.abbreviate import abbreviate_space, abbreviate_time from allmydata.util.encodingutil import argv_to_abspath +if PY2: + _print = print + def print(*args, **kwargs): + """ + Builtin ``print``-alike that will even write unicode not encodeable using + the specified output file's encoding. + """ + out = kwargs.pop("file", None) + if out is None: + from sys import stdout as out + encoding = out.encoding or "ascii" + def ensafe(o): + if isinstance(o, unicode): + return o.encode(encoding, errors="replace").decode(encoding) + return o + return _print( + *(ensafe(a) for a in args), + file=out, + **kwargs + ) def _get_request_parameters_for_fragment(options, fragment, method, post_args): """ From a988f126bee53252c0692dda40b3c0262000cba4 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 7 Sep 2021 16:12:01 -0400 Subject: [PATCH 129/269] fix markup error --- docs/proposed/http-storage-node-protocol.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index 5f67fcaa6..a84d62176 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -539,8 +539,8 @@ For example:: [1, 5] -``GET /v1/immutable/:storage_index/:share_number -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +``GET /v1/immutable/:storage_index/:share_number`` +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! Read a contiguous sequence of bytes from one share in one bucket. The response body is the raw share data (i.e., ``application/octet-stream``). From 8ca1f8c33511d64b26de17d7356ae11c88be6ecd Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Tue, 7 Sep 2021 16:19:15 -0400 Subject: [PATCH 130/269] Python 3 is grumpy that it doesn't get to play Fool it --- src/allmydata/scripts/tahoe_status.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/allmydata/scripts/tahoe_status.py b/src/allmydata/scripts/tahoe_status.py index 9b2c89d24..45fbdb9ba 100644 --- a/src/allmydata/scripts/tahoe_status.py +++ b/src/allmydata/scripts/tahoe_status.py @@ -28,6 +28,7 @@ if PY2: Builtin ``print``-alike that will even write unicode not encodeable using the specified output file's encoding. """ + from past.builtins import unicode out = kwargs.pop("file", None) if out is None: from sys import stdout as out From e0414fd8af374ab8b4f04cbeb8d38be94d18b16a Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 8 Sep 2021 09:14:47 -0400 Subject: [PATCH 131/269] Upload eliot.log on CircleCI runs --- .circleci/config.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 28e4c8d58..62d1bd752 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -271,6 +271,11 @@ jobs: # in the project source checkout. path: "/tmp/project/_trial_temp/test.log" + - store_artifacts: &STORE_ELIOT_LOG + # Despite passing --workdir /tmp to tox above, it still runs trial + # in the project source checkout. + path: "/tmp/project/eliot.log" + - store_artifacts: &STORE_OTHER_ARTIFACTS # Store any other artifacts, too. This is handy to allow other jobs # sharing most of the definition of this one to be able to @@ -413,6 +418,7 @@ jobs: - run: *RUN_TESTS - store_test_results: *STORE_TEST_RESULTS - store_artifacts: *STORE_TEST_LOG + - store_artifacts: *STORE_ELIOT_LOG - store_artifacts: *STORE_OTHER_ARTIFACTS - run: *SUBMIT_COVERAGE From bcf1c7153676971d75be573f2cf9ad64c360f14a Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 8 Sep 2021 09:15:02 -0400 Subject: [PATCH 132/269] Upload trial's test.log on GitHub Actions --- .github/workflows/ci.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e95d2ee88..f4e0b50ce 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -83,6 +83,12 @@ jobs: name: eliot.log path: eliot.log + - name: Upload trial log + uses: actions/upload-artifact@v1 + with: + name: _trial_temp/test.log + path: _trial_temp/test.log + # Upload this job's coverage data to Coveralls. While there is a GitHub # Action for this, as of Jan 2021 it does not support Python coverage # files - only lcov files. Therefore, we use coveralls-python, the From 1d4bc54b487e07d0c7316da77b834a2ddf2bc856 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 8 Sep 2021 09:15:13 -0400 Subject: [PATCH 133/269] Just upload the log all the time --- .github/workflows/ci.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f4e0b50ce..9e6c246b6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -76,9 +76,8 @@ jobs: - name: Run tox for corresponding Python version run: python -m tox - - name: Upload eliot.log in case of failure + - name: Upload eliot.log uses: actions/upload-artifact@v1 - if: failure() with: name: eliot.log path: eliot.log From dd4e6c7741deb2545400b9c0bcb1337edccf483c Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 8 Sep 2021 09:15:28 -0400 Subject: [PATCH 134/269] whitespace --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9e6c246b6..cd6f872bb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -141,7 +141,7 @@ jobs: # See notes about parallel builds on GitHub Actions at # https://coveralls-python.readthedocs.io/en/latest/usage/configuration.html finish-coverage-report: - needs: + needs: - "coverage" runs-on: "ubuntu-latest" container: "python:3-slim" @@ -178,7 +178,7 @@ jobs: - name: Install Tor [Ubuntu] if: matrix.os == 'ubuntu-latest' run: sudo apt install tor - + # TODO: See https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3744. # We have to use an older version of Tor for running integration # tests on macOS. From e3804e0354f30c4fc9a1371e5c4d9f5fc52b6a3a Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 8 Sep 2021 09:15:40 -0400 Subject: [PATCH 135/269] news fragment --- newsfragments/3792.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3792.minor diff --git a/newsfragments/3792.minor b/newsfragments/3792.minor new file mode 100644 index 000000000..e69de29bb From 93f2cf5ce1460192cae4c3ba9d42808c2caed107 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 8 Sep 2021 09:41:32 -0400 Subject: [PATCH 136/269] Minor edits --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index bb8c5c252..3686a58b8 100644 --- a/README.rst +++ b/README.rst @@ -56,10 +56,10 @@ For more detailed instructions, read `docs/INSTALL.rst `__ . Once tahoe --version works, see `docs/running.rst `__ to learn how to set up your first Tahoe-LAFS node. -🐍 `Python 3` Support +🐍 Python 3 Support -------------------- -`Python 3` support has been introduced as from `Tahoe-LAFS 1.16.x` alongside `Python 2`. System admnistrators are advised to start running Tahoe on `Python 3` and should expect the drop of `Python 2` support any future version from now. Please, feel free to file issues if you run into bugs while running Tahoe on `Python 3`. +Python 3 support has been introduced starting with Tahoe-LAFS 1.16.0, alongside Python 2. System admnistrators are advised to start running Tahoe on Python 3 and should expect Python 2 support to be dropped in a future version. Please, feel free to file issues if you run into bugs while running Tahoe on Python 3. 🤖 Issues From 2a6870d77253f0274b56cc9f9d725d7bd305b007 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 8 Sep 2021 09:42:23 -0400 Subject: [PATCH 137/269] The name must be pathless, it seems --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cd6f872bb..e161ec243 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -85,7 +85,7 @@ jobs: - name: Upload trial log uses: actions/upload-artifact@v1 with: - name: _trial_temp/test.log + name: test.log path: _trial_temp/test.log # Upload this job's coverage data to Coveralls. While there is a GitHub From edb380f80163f87428f5ba9cbaa1904d099228c4 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 8 Sep 2021 10:48:32 -0400 Subject: [PATCH 138/269] Bridge Foolscap logs to Twisted's so they appear in test.log --- src/allmydata/test/__init__.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/__init__.py b/src/allmydata/test/__init__.py index 893aa15ce..8b11ffd17 100644 --- a/src/allmydata/test/__init__.py +++ b/src/allmydata/test/__init__.py @@ -55,9 +55,22 @@ def disable_foolscap_incidents(): iq = NonQualifier() theLogger.setIncidentQualifier(iq) -# we disable incident reporting for all unit tests. -disable_foolscap_incidents() +def bridge_foolscap_logs_to_twisted(): + # Dump all of the Foolscap logs into the Twisted logging system where they + # can get scooped up by any other log observers we configure. + from foolscap.logging import log + log.bridgeLogsToTwisted() +def configure_foolscap_logging(): + # we disable incident reporting for all unit tests. + disable_foolscap_incidents() + + # we want to collect Foolscap logs too, and it's easiest to do this by + # getting them into one of the log systems we already have collection set + # up for. + bridge_foolscap_logs_to_twisted() + +configure_foolscap_logging() def _configure_hypothesis(): from os import environ From 2dbb9434b0d10b086e6d5367087aa1a4cdd77be0 Mon Sep 17 00:00:00 2001 From: Florian Sesser Date: Wed, 8 Sep 2021 14:54:57 +0000 Subject: [PATCH 139/269] OpenMetrics endpoint WIP --- src/allmydata/web/status.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/src/allmydata/web/status.py b/src/allmydata/web/status.py index 741f93061..7d3fdd06e 100644 --- a/src/allmydata/web/status.py +++ b/src/allmydata/web/status.py @@ -1554,6 +1554,10 @@ class Statistics(MultiFormatResource): @render_exception def render_OPENMETRICS(self, req): + req.setHeader("content-type", "application/openmetrics-text; version=1.0.0; charset=utf-8") + stats = self._provider.get_stats() + ret = u"" + def mangle_name(name): return re.sub( "_(\d\d)_(\d)_percentile", @@ -1561,15 +1565,13 @@ class Statistics(MultiFormatResource): name.replace(".", "_") ) - req.setHeader( - "content-type", "application/openmetrics-text; version=1.0.0; charset=utf-8" - ) + for (k, v) in sorted(stats['counters'].items()): + ret += u"tahoe_counters_%s %s\n" % (mangle_name(k), v) - stats = self._provider.get_stats() - return (str({mangle_name(k): v for k, v in stats['counters'].items()}) - + str({mangle_name(k): v for k, v in stats['stats'].items()}) - + "\n" - ) + for (k, v) in sorted(stats['stats'].items()): + ret += u"tahoe_stats_%s %s\n" % (mangle_name(k), v) + + return ret class StatisticsElement(Element): From ca865e60db6d94bba6abb749a2db196a77b25d36 Mon Sep 17 00:00:00 2001 From: Florian Sesser Date: Wed, 8 Sep 2021 15:08:25 +0000 Subject: [PATCH 140/269] OpenMetrics endpoint --- src/allmydata/web/status.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/allmydata/web/status.py b/src/allmydata/web/status.py index 7d3fdd06e..4be935a54 100644 --- a/src/allmydata/web/status.py +++ b/src/allmydata/web/status.py @@ -1565,12 +1565,13 @@ class Statistics(MultiFormatResource): name.replace(".", "_") ) + def mangle_value(val): + return str(val) if val is not None else "NaN" + for (k, v) in sorted(stats['counters'].items()): - ret += u"tahoe_counters_%s %s\n" % (mangle_name(k), v) - + ret += u"tahoe_counters_%s %s\n" % (mangle_name(k), mangle_value(v)) for (k, v) in sorted(stats['stats'].items()): - ret += u"tahoe_stats_%s %s\n" % (mangle_name(k), v) - + ret += u"tahoe_stats_%s %s\n" % (mangle_name(k), mangle_value(v)) return ret class StatisticsElement(Element): From 855d02bef0587c7517417ac0528c1f65ae68111c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 8 Sep 2021 11:26:52 -0400 Subject: [PATCH 141/269] Start thinking about immutable tests. --- src/allmydata/test/test_istorageserver.py | 51 ++++++++++++++++++----- 1 file changed, 41 insertions(+), 10 deletions(-) diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index 51d5da2c0..b2229f467 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -1,38 +1,69 @@ """ Tests for the ``IStorageServer`` interface. + +Note that for performance, in the future we might want the same node to be +reused across tests, so each test should be careful to generate unique storage +indexes. """ -from twisted.trial import unittest -from twisted.internet.defer import inlineCallbacks +from twisted.internet.defer import inlineCallbacks, returnValue +from twisted.trial.unittest import TestCase from allmydata.interfaces import IStorageServer from .test_system import SystemTestMixin -class IStorageServerTestsMixin: +class IStorageServerSharedAPIsTestsMixin(object): """ - Tests for ``IStorageServer``. + Tests for ``IStorageServer``'s shared APIs. ``self.storage_server`` is expected to provide ``IStorageServer``. """ + @inlineCallbacks def test_version(self): + # TODO get_version() returns a dict-like thing with some of the + # expected fields. yield self.storage_server.get_version() -class FoolscapIStorageServerTests( - SystemTestMixin, IStorageServerTestsMixin, unittest.TestCase -): +class IStorageServerImmutableAPIsTestsMixin(object): + """ + Tests for ``IStorageServer``'s immutable APIs. + + ``self.storage_server`` is expected to provide ``IStorageServer``. + """ + + # TODO === allocate_buckets + RIBucketWriter === + # TODO allocate_buckets on a new storage index + # TODO allocate_buckets on existing bucket with same sharenums + # TODO allocate_buckets with smaller sharenums + # TODO allocate_buckets with larger sharenums + # TODO writes to bucket can happen in any order (write then read) + # TODO overlapping writes ignore already-written data (write then read) + + +class _FoolscapMixin(SystemTestMixin): """Run tests on Foolscap version of ``IStorageServer.""" @inlineCallbacks def setUp(self): - self.basedir = "test_istorageserver/{}/{}".format( - self.__class__.__name__, self._testMethodName - ) + self.basedir = "test_istorageserver/" + self.id() yield SystemTestMixin.setUp(self) yield self.set_up_nodes(1) self.storage_server = next( iter(self.clients[0].storage_broker.get_known_servers()) ).get_storage_server() self.assertTrue(IStorageServer.providedBy(self.storage_server)) + + +class FoolscapSharedAPIsTests( + _FoolscapMixin, IStorageServerSharedAPIsTestsMixin, TestCase +): + """Foolscap-specific tests for shared ``IStorageServer`` APIs.""" + + +class FoolscapImmutableAPIsTests( + _FoolscapMixin, IStorageServerImmutableAPIsTestsMixin, TestCase +): + """Foolscap-specific tests for immutable ``IStorageServer`` APIs.""" From 3bec2a480f8d17e8e22cbd23fac2b4611a786e36 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 8 Sep 2021 12:20:27 -0400 Subject: [PATCH 142/269] Start on allocate_bucket tests. --- src/allmydata/test/test_istorageserver.py | 152 +++++++++++++++++++++- 1 file changed, 149 insertions(+), 3 deletions(-) diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index b2229f467..303b4b997 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -6,13 +6,68 @@ reused across tests, so each test should be careful to generate unique storage indexes. """ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from future.utils import PY2 + +if PY2: + from future.builtins import ( + filter, + map, + zip, + ascii, + chr, + hex, + input, + next, + oct, + open, + pow, + round, + super, + bytes, + dict, + list, + object, + range, + str, + max, + min, + ) # noqa: F401 + +from random import randrange +from unittest import expectedFailure + from twisted.internet.defer import inlineCallbacks, returnValue from twisted.trial.unittest import TestCase -from allmydata.interfaces import IStorageServer +from foolscap.api import Referenceable + +from allmydata.interfaces import IStorageServer, RIBucketWriter from .test_system import SystemTestMixin +def _randbytes(length): + # type: (int) -> bytes + """Return random bytes string of given length.""" + return bytes([randrange(0, 256) for _ in range(length)]) + + +def new_storage_index(): + # type: () -> bytes + """Return a new random storage index.""" + return _randbytes(16) + + +def new_secret(): + # type: () -> bytes + """Return a new random secret (for lease renewal or cancellation).""" + return _randbytes(32) + + class IStorageServerSharedAPIsTestsMixin(object): """ Tests for ``IStorageServer``'s shared APIs. @@ -35,13 +90,104 @@ class IStorageServerImmutableAPIsTestsMixin(object): """ # TODO === allocate_buckets + RIBucketWriter === - # TODO allocate_buckets on a new storage index - # TODO allocate_buckets on existing bucket with same sharenums + # DONE allocate_buckets on a new storage index + # PROG allocate_buckets on existing bucket with same sharenums # TODO allocate_buckets with smaller sharenums # TODO allocate_buckets with larger sharenums # TODO writes to bucket can happen in any order (write then read) # TODO overlapping writes ignore already-written data (write then read) + @inlineCallbacks + def test_allocate_buckets_new(self): + """ + allocate_buckets() with a new storage index returns the matching + shares. + """ + (already_got, allocated) = yield self.storage_server.allocate_buckets( + new_storage_index(), + new_secret(), + new_secret(), + set(range(5)), + 1024, + Referenceable(), + ) + self.assertEqual(already_got, set()) + self.assertEqual(allocated.keys(), set(range(5))) + # We validate the bucket objects' interface in a later test. + + @inlineCallbacks + def test_allocate_buckets_repeat(self): + """ + allocate_buckets() with the same storage index returns the same result, + because the shares have not been written to. + + This fails due to https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3793 + """ + si, renew_secret, cancel_secret = ( + new_storage_index(), + new_secret(), + new_secret(), + ) + (already_got, allocated) = yield self.storage_server.allocate_buckets( + si, + renew_secret, + cancel_secret, + set(range(5)), + 1024, + Referenceable(), + ) + (already_got2, allocated2) = yield self.storage_server.allocate_buckets( + si, + renew_secret, + cancel_secret, + set(range(5)), + 1024, + Referenceable(), + ) + self.assertEqual(already_got, already_got2) + self.assertEqual(allocated.keys(), allocated2.keys()) + + test_allocate_buckets_repeat.todo = ( + "https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3793" + ) + + @expectedFailure + @inlineCallbacks + def test_allocate_buckets_more_sharenums(self): + """ + allocate_buckets() with the same storage index but more sharenums + acknowledges the extra shares don't exist. + + Fails due to https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3793 + """ + si, renew_secret, cancel_secret = ( + new_storage_index(), + new_secret(), + new_secret(), + ) + yield self.storage_server.allocate_buckets( + si, + renew_secret, + cancel_secret, + set(range(5)), + 1024, + Referenceable(), + ) + (already_got2, allocated2) = yield self.storage_server.allocate_buckets( + si, + renew_secret, + cancel_secret, + set(range(7)), + 1024, + Referenceable(), + ) + self.assertEqual(already_got2, set()) # none were fully written + self.assertEqual(allocated2.keys(), set(range(7))) + + test_allocate_buckets_more_sharenums.todo = ( + "https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3793" + ) + class _FoolscapMixin(SystemTestMixin): """Run tests on Foolscap version of ``IStorageServer.""" From 30b43a8ed6b543a5fe8e0dc01d8cbebb175cd7bd Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 8 Sep 2021 12:41:37 -0400 Subject: [PATCH 143/269] Revert "Bridge Foolscap logs to Twisted's so they appear in test.log" This reverts commit edb380f80163f87428f5ba9cbaa1904d099228c4. This breaks the test suite hard --- src/allmydata/test/__init__.py | 17 ++--------------- 1 file changed, 2 insertions(+), 15 deletions(-) diff --git a/src/allmydata/test/__init__.py b/src/allmydata/test/__init__.py index 8b11ffd17..893aa15ce 100644 --- a/src/allmydata/test/__init__.py +++ b/src/allmydata/test/__init__.py @@ -55,22 +55,9 @@ def disable_foolscap_incidents(): iq = NonQualifier() theLogger.setIncidentQualifier(iq) -def bridge_foolscap_logs_to_twisted(): - # Dump all of the Foolscap logs into the Twisted logging system where they - # can get scooped up by any other log observers we configure. - from foolscap.logging import log - log.bridgeLogsToTwisted() +# we disable incident reporting for all unit tests. +disable_foolscap_incidents() -def configure_foolscap_logging(): - # we disable incident reporting for all unit tests. - disable_foolscap_incidents() - - # we want to collect Foolscap logs too, and it's easiest to do this by - # getting them into one of the log systems we already have collection set - # up for. - bridge_foolscap_logs_to_twisted() - -configure_foolscap_logging() def _configure_hypothesis(): from os import environ From 88d3ee5785644229870086910ef9c969c2d16c50 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 8 Sep 2021 13:43:47 -0400 Subject: [PATCH 144/269] debug print --- src/allmydata/scripts/tahoe_status.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/allmydata/scripts/tahoe_status.py b/src/allmydata/scripts/tahoe_status.py index 45fbdb9ba..4817c7b5a 100644 --- a/src/allmydata/scripts/tahoe_status.py +++ b/src/allmydata/scripts/tahoe_status.py @@ -37,6 +37,7 @@ if PY2: if isinstance(o, unicode): return o.encode(encoding, errors="replace").decode(encoding) return o + _print("printing {!r} with encoding {!r}".format(args, encoding)) return _print( *(ensafe(a) for a in args), file=out, From c1b1ed0dc34950b6931af575814e436ad9d0c8d1 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 8 Sep 2021 13:52:34 -0400 Subject: [PATCH 145/269] More tests. --- src/allmydata/test/test_istorageserver.py | 105 ++++++++++++++++++++-- 1 file changed, 97 insertions(+), 8 deletions(-) diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index 303b4b997..92217debc 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -89,14 +89,6 @@ class IStorageServerImmutableAPIsTestsMixin(object): ``self.storage_server`` is expected to provide ``IStorageServer``. """ - # TODO === allocate_buckets + RIBucketWriter === - # DONE allocate_buckets on a new storage index - # PROG allocate_buckets on existing bucket with same sharenums - # TODO allocate_buckets with smaller sharenums - # TODO allocate_buckets with larger sharenums - # TODO writes to bucket can happen in any order (write then read) - # TODO overlapping writes ignore already-written data (write then read) - @inlineCallbacks def test_allocate_buckets_new(self): """ @@ -188,6 +180,103 @@ class IStorageServerImmutableAPIsTestsMixin(object): "https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3793" ) + @inlineCallbacks + def test_written_shares_are_allocated(self): + """ + Shares that are fully written to show up as allocated in result from + ``IStoragServer.allocate_buckets()``. Partially-written or empty + shares don't. + """ + si, renew_secret, cancel_secret = ( + new_storage_index(), + new_secret(), + new_secret(), + ) + (_, allocated) = yield self.storage_server.allocate_buckets( + si, + renew_secret, + cancel_secret, + set(range(5)), + 1024, + Referenceable(), + ) + + # Bucket 1 is fully written in one go. + yield allocated[1].callRemote("write", 0, b"1" * 1024) + yield allocated[1].callRemote("close") + + # Bucket 2 is fully written in two steps. + yield allocated[2].callRemote("write", 0, b"1" * 512) + yield allocated[2].callRemote("write", 512, b"2" * 512) + yield allocated[2].callRemote("close") + + # Bucket 0 has partial write. + yield allocated[0].callRemote("write", 0, b"1" * 512) + + (already_got, _) = yield self.storage_server.allocate_buckets( + si, + renew_secret, + cancel_secret, + set(range(5)), + 1024, + Referenceable(), + ) + self.assertEqual(already_got, {1, 2}) + + @inlineCallbacks + def test_written_shares_are_readable(self): + """ + Shares that are fully written to can be read. + + 1. The result is not affected by the order in which writes + happened, only by their offsets. + + 2. When overlapping writes happen, the resulting read returns the + earliest written value. + """ + si, renew_secret, cancel_secret = ( + new_storage_index(), + new_secret(), + new_secret(), + ) + (_, allocated) = yield self.storage_server.allocate_buckets( + si, + renew_secret, + cancel_secret, + set(range(5)), + 1024, + Referenceable(), + ) + + # Bucket 1 is fully written in order + yield allocated[1].callRemote("write", 0, b"1" * 512) + yield allocated[1].callRemote("write", 512, b"2" * 512) + yield allocated[1].callRemote("close") + + # Bucket 2 is fully written in reverse. + yield allocated[2].callRemote("write", 512, b"4" * 512) + yield allocated[2].callRemote("write", 0, b"3" * 512) + yield allocated[2].callRemote("close") + + # Bucket 3 has an overlapping write. + yield allocated[3].callRemote("write", 0, b"5" * 20) + yield allocated[3].callRemote("write", 0, b"5" * 24) + yield allocated[3].callRemote("write", 24, b"6" * 1000) + yield allocated[3].callRemote("close") + + buckets = yield self.storage_server.get_buckets(si) + self.assertEqual(buckets.keys(), {1, 2, 3}) + + self.assertEqual( + (yield buckets[1].callRemote("read", 0, 1024)), b"1" * 512 + b"2" * 512 + ) + self.assertEqual( + (yield buckets[2].callRemote("read", 0, 1024)), b"3" * 512 + b"4" * 512 + ) + self.assertEqual( + (yield buckets[3].callRemote("read", 0, 1024)), b"5" * 24 + b"6" * 1000 + ) + class _FoolscapMixin(SystemTestMixin): """Run tests on Foolscap version of ``IStorageServer.""" From f3cb42d9a847ca8fa4a52fc9b69e2b4161a35071 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 8 Sep 2021 13:52:51 -0400 Subject: [PATCH 146/269] News file. --- newsfragments/3784.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3784.minor diff --git a/newsfragments/3784.minor b/newsfragments/3784.minor new file mode 100644 index 000000000..e69de29bb From 44388037dfe034130829ca830ed2f5d39ff9f908 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 8 Sep 2021 14:12:32 -0400 Subject: [PATCH 147/269] Flakes, and closer to passing on Python 2. --- src/allmydata/test/test_istorageserver.py | 44 ++++++----------------- 1 file changed, 11 insertions(+), 33 deletions(-) diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index 92217debc..68d7e2972 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -11,49 +11,28 @@ from __future__ import division from __future__ import print_function from __future__ import unicode_literals -from future.utils import PY2 +from future.utils import PY2, bchr if PY2: - from future.builtins import ( - filter, - map, - zip, - ascii, - chr, - hex, - input, - next, - oct, - open, - pow, - round, - super, - bytes, - dict, - list, - object, - range, - str, - max, - min, - ) # noqa: F401 + # fmt: off + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + # fmt: on from random import randrange -from unittest import expectedFailure -from twisted.internet.defer import inlineCallbacks, returnValue +from twisted.internet.defer import inlineCallbacks from twisted.trial.unittest import TestCase from foolscap.api import Referenceable -from allmydata.interfaces import IStorageServer, RIBucketWriter +from allmydata.interfaces import IStorageServer from .test_system import SystemTestMixin def _randbytes(length): # type: (int) -> bytes """Return random bytes string of given length.""" - return bytes([randrange(0, 256) for _ in range(length)]) + return b"".join([bchr(randrange(0, 256)) for _ in range(length)]) def new_storage_index(): @@ -104,7 +83,7 @@ class IStorageServerImmutableAPIsTestsMixin(object): Referenceable(), ) self.assertEqual(already_got, set()) - self.assertEqual(allocated.keys(), set(range(5))) + self.assertEqual(set(allocated.keys()), set(range(5))) # We validate the bucket objects' interface in a later test. @inlineCallbacks @@ -137,13 +116,12 @@ class IStorageServerImmutableAPIsTestsMixin(object): Referenceable(), ) self.assertEqual(already_got, already_got2) - self.assertEqual(allocated.keys(), allocated2.keys()) + self.assertEqual(set(allocated.keys()), set(allocated2.keys())) test_allocate_buckets_repeat.todo = ( "https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3793" ) - @expectedFailure @inlineCallbacks def test_allocate_buckets_more_sharenums(self): """ @@ -174,7 +152,7 @@ class IStorageServerImmutableAPIsTestsMixin(object): Referenceable(), ) self.assertEqual(already_got2, set()) # none were fully written - self.assertEqual(allocated2.keys(), set(range(7))) + self.assertEqual(set(allocated2.keys()), set(range(7))) test_allocate_buckets_more_sharenums.todo = ( "https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3793" @@ -265,7 +243,7 @@ class IStorageServerImmutableAPIsTestsMixin(object): yield allocated[3].callRemote("close") buckets = yield self.storage_server.get_buckets(si) - self.assertEqual(buckets.keys(), {1, 2, 3}) + self.assertEqual(set(buckets.keys()), {1, 2, 3}) self.assertEqual( (yield buckets[1].callRemote("read", 0, 1024)), b"1" * 512 + b"2" * 512 From a2d54aa8bc4cfc7aaf40a5c3f29246d51606fce9 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 8 Sep 2021 14:14:36 -0400 Subject: [PATCH 148/269] .todo isn't working on Python 2 for some reason. --- src/allmydata/test/test_istorageserver.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index 68d7e2972..75fbe42e2 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -118,7 +118,7 @@ class IStorageServerImmutableAPIsTestsMixin(object): self.assertEqual(already_got, already_got2) self.assertEqual(set(allocated.keys()), set(allocated2.keys())) - test_allocate_buckets_repeat.todo = ( + test_allocate_buckets_repeat.skip = ( "https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3793" ) @@ -154,7 +154,7 @@ class IStorageServerImmutableAPIsTestsMixin(object): self.assertEqual(already_got2, set()) # none were fully written self.assertEqual(set(allocated2.keys()), set(range(7))) - test_allocate_buckets_more_sharenums.todo = ( + test_allocate_buckets_more_sharenums.skip = ( "https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3793" ) From 19086a63487d340909040be635a8d66dc50881d8 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 8 Sep 2021 14:25:10 -0400 Subject: [PATCH 149/269] log sys.stdout.encoding from tox env --- tox.ini | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tox.ini b/tox.ini index 9b0f71038..af10270ca 100644 --- a/tox.ini +++ b/tox.ini @@ -74,6 +74,8 @@ commands = tahoe --version + python -c "import sys; print('sys.stdout.encoding: {}'.format(sys.stdout.encoding))" + # Run tests with -b to catch bugs like `"%s" % (some_bytes,)`. -b makes # Python emit BytesWarnings, and warnings configuration in # src/allmydata/tests/__init__.py turns allmydata's BytesWarnings into From 3e3fd1aa75d83b4429a6f7c6a9296936b03aaca1 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 8 Sep 2021 14:25:30 -0400 Subject: [PATCH 150/269] on Windows, Python 3 wants to play --- src/allmydata/scripts/tahoe_status.py | 42 +++++++++++++-------------- 1 file changed, 20 insertions(+), 22 deletions(-) diff --git a/src/allmydata/scripts/tahoe_status.py b/src/allmydata/scripts/tahoe_status.py index 4817c7b5a..e73958642 100644 --- a/src/allmydata/scripts/tahoe_status.py +++ b/src/allmydata/scripts/tahoe_status.py @@ -21,28 +21,26 @@ from allmydata.scripts.common_http import BadResponse from allmydata.util.abbreviate import abbreviate_space, abbreviate_time from allmydata.util.encodingutil import argv_to_abspath -if PY2: - _print = print - def print(*args, **kwargs): - """ - Builtin ``print``-alike that will even write unicode not encodeable using - the specified output file's encoding. - """ - from past.builtins import unicode - out = kwargs.pop("file", None) - if out is None: - from sys import stdout as out - encoding = out.encoding or "ascii" - def ensafe(o): - if isinstance(o, unicode): - return o.encode(encoding, errors="replace").decode(encoding) - return o - _print("printing {!r} with encoding {!r}".format(args, encoding)) - return _print( - *(ensafe(a) for a in args), - file=out, - **kwargs - ) +_print = print +def print(*args, **kwargs): + """ + Builtin ``print``-alike that will even write unicode not encodeable using + the specified output file's encoding. + """ + from past.builtins import unicode + out = kwargs.pop("file", None) + if out is None: + from sys import stdout as out + encoding = out.encoding or "ascii" + def ensafe(o): + if isinstance(o, unicode): + return o.encode(encoding, errors="replace").decode(encoding) + return o + return _print( + *(ensafe(a) for a in args), + file=out, + **kwargs + ) def _get_request_parameters_for_fragment(options, fragment, method, post_args): """ From 903da3d513664f96e4b2936abc6a5287e56af2c2 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 8 Sep 2021 14:35:55 -0400 Subject: [PATCH 151/269] Fix the ... tox.ini ... syntax error ... --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index af10270ca..f40f6f3ba 100644 --- a/tox.ini +++ b/tox.ini @@ -74,7 +74,7 @@ commands = tahoe --version - python -c "import sys; print('sys.stdout.encoding: {}'.format(sys.stdout.encoding))" + python -c "import sys; print('sys.stdout.encoding:', sys.stdout.encoding)" # Run tests with -b to catch bugs like `"%s" % (some_bytes,)`. -b makes # Python emit BytesWarnings, and warnings configuration in From 4674bccde799be427c20dcfa8cc133126b620b99 Mon Sep 17 00:00:00 2001 From: Florian Sesser Date: Thu, 9 Sep 2021 13:54:03 +0000 Subject: [PATCH 152/269] OpenMetrics: add trailing EOF marker --- src/allmydata/web/status.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/allmydata/web/status.py b/src/allmydata/web/status.py index 4be935a54..3e0e1a3ee 100644 --- a/src/allmydata/web/status.py +++ b/src/allmydata/web/status.py @@ -1572,6 +1572,9 @@ class Statistics(MultiFormatResource): ret += u"tahoe_counters_%s %s\n" % (mangle_name(k), mangle_value(v)) for (k, v) in sorted(stats['stats'].items()): ret += u"tahoe_stats_%s %s\n" % (mangle_name(k), mangle_value(v)) + + ret += u"# EOF\n" + return ret class StatisticsElement(Element): From d05e373d4236f7e5a106120da11882b64abd7ded Mon Sep 17 00:00:00 2001 From: Florian Sesser Date: Thu, 9 Sep 2021 13:57:59 +0000 Subject: [PATCH 153/269] OpenMetrics: All strings are unicode. --- src/allmydata/web/status.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/allmydata/web/status.py b/src/allmydata/web/status.py index 3e0e1a3ee..6d36861ad 100644 --- a/src/allmydata/web/status.py +++ b/src/allmydata/web/status.py @@ -1560,13 +1560,13 @@ class Statistics(MultiFormatResource): def mangle_name(name): return re.sub( - "_(\d\d)_(\d)_percentile", - '{quantile="0.\g<1>\g<2>"}', - name.replace(".", "_") + u"_(\d\d)_(\d)_percentile", + u'{quantile="0.\g<1>\g<2>"}', + name.replace(u".", u"_") ) def mangle_value(val): - return str(val) if val is not None else "NaN" + return str(val) if val is not None else u"NaN" for (k, v) in sorted(stats['counters'].items()): ret += u"tahoe_counters_%s %s\n" % (mangle_name(k), mangle_value(v)) From 74965b271c026fe35436207a3d358ff9c40b32dc Mon Sep 17 00:00:00 2001 From: Florian Sesser Date: Thu, 9 Sep 2021 22:36:30 +0000 Subject: [PATCH 154/269] typo --- newsfragments/3786.feature | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/newsfragments/3786.feature b/newsfragments/3786.feature index 82ce3f974..ecbfc0372 100644 --- a/newsfragments/3786.feature +++ b/newsfragments/3786.feature @@ -1 +1 @@ -tahoe-lafs now provides its statistics also in OpenMetrics format (for Prometheus et. al.) on `/statistics?t=openmetrics`. +tahoe-lafs now provides its statistics also in OpenMetrics format (for Prometheus et. al.) at `/statistics?t=openmetrics`. From 30771149fc7467fd7a61076b1572bcfddaa9a218 Mon Sep 17 00:00:00 2001 From: Florian Sesser Date: Thu, 9 Sep 2021 23:31:39 +0000 Subject: [PATCH 155/269] Openmetrics: Add test case scaffold --- src/allmydata/test/test_openmetrics.py | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 src/allmydata/test/test_openmetrics.py diff --git a/src/allmydata/test/test_openmetrics.py b/src/allmydata/test/test_openmetrics.py new file mode 100644 index 000000000..ad11f3468 --- /dev/null +++ b/src/allmydata/test/test_openmetrics.py @@ -0,0 +1,11 @@ +from twisted.trial import unittest + +class FakeStatsProvider(object): + def get_stats(self): + stats = {'stats': {}, 'counters': {}} + return stats + +class OpenMetrics(unittest.TestCase): + def test_spec_compliance(self): + self.assertEqual('1', '2') + From fca1482b35b2360d07ec700c0a22d20162bc5acb Mon Sep 17 00:00:00 2001 From: Florian Sesser Date: Fri, 10 Sep 2021 00:10:11 +0000 Subject: [PATCH 156/269] OpenMetrics Tests WIP --- src/allmydata/test/test_openmetrics.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/test_openmetrics.py b/src/allmydata/test/test_openmetrics.py index ad11f3468..2211c3561 100644 --- a/src/allmydata/test/test_openmetrics.py +++ b/src/allmydata/test/test_openmetrics.py @@ -1,4 +1,6 @@ +import mock from twisted.trial import unittest +from allmydata.web.status import Statistics class FakeStatsProvider(object): def get_stats(self): @@ -6,6 +8,18 @@ class FakeStatsProvider(object): return stats class OpenMetrics(unittest.TestCase): - def test_spec_compliance(self): - self.assertEqual('1', '2') + def test_header(self): + req = mock.Mock() + stats = mock.Mock() + stats._provider = FakeStatsProvider() + metrics = Statistics.render_OPENMETRICS(stats, req) + req.setHeader.assert_called_with("content-type", "application/openmetrics-text; version=1.0.0; charset=utf-8") + + def test_spec_compliance(self): + req = mock.Mock() + stats = mock.Mock() + stats._provider = FakeStatsProvider() + metrics = Statistics.render_OPENMETRICS(stats, req) + # TODO test that output adheres to spec + # TODO add more realistic stats, incl. missing (None) values From d04157d18a0fbe9a1832f94bf352b04e931039e1 Mon Sep 17 00:00:00 2001 From: Florian Sesser Date: Fri, 10 Sep 2021 13:00:15 +0000 Subject: [PATCH 157/269] OpenMetrics test: Add parser to check against spec --- src/allmydata/test/test_openmetrics.py | 18 +++++++++++------- tox.ini | 2 ++ 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/src/allmydata/test/test_openmetrics.py b/src/allmydata/test/test_openmetrics.py index 2211c3561..cd5c5d407 100644 --- a/src/allmydata/test/test_openmetrics.py +++ b/src/allmydata/test/test_openmetrics.py @@ -1,4 +1,5 @@ import mock +from prometheus_client.openmetrics import parser from twisted.trial import unittest from allmydata.web.status import Statistics @@ -8,18 +9,21 @@ class FakeStatsProvider(object): return stats class OpenMetrics(unittest.TestCase): - def test_header(self): + def test_spec_compliance(self): + """ + Does our output adhere to the OpenMetrics spec? + https://github.com/OpenObservability/OpenMetrics/blob/main/specification/OpenMetrics.md + """ req = mock.Mock() stats = mock.Mock() stats._provider = FakeStatsProvider() metrics = Statistics.render_OPENMETRICS(stats, req) + + # "The content type MUST be..." req.setHeader.assert_called_with("content-type", "application/openmetrics-text; version=1.0.0; charset=utf-8") - def test_spec_compliance(self): - req = mock.Mock() - stats = mock.Mock() - stats._provider = FakeStatsProvider() - metrics = Statistics.render_OPENMETRICS(stats, req) - # TODO test that output adheres to spec + # The parser throws if it can't parse. + # Wrap in a list() to drain the generator. + families = list(parser.text_string_to_metric_families(metrics)) # TODO add more realistic stats, incl. missing (None) values diff --git a/tox.ini b/tox.ini index 9b0f71038..0e8e58ea6 100644 --- a/tox.ini +++ b/tox.ini @@ -52,6 +52,8 @@ deps = certifi # VCS hooks support py36,!coverage: pre-commit + # Does our OpenMetrics endpoint adhere to the spec: + prometheus-client==0.11.0 # We add usedevelop=False because testing against a true installation gives # more useful results. From 6c18983f7b182fe696dd466dab517312692c1648 Mon Sep 17 00:00:00 2001 From: Florian Sesser Date: Fri, 10 Sep 2021 13:13:13 +0000 Subject: [PATCH 158/269] OpenMetrics test: Use realistic input data --- src/allmydata/test/test_openmetrics.py | 92 +++++++++++++++++++++++++- 1 file changed, 89 insertions(+), 3 deletions(-) diff --git a/src/allmydata/test/test_openmetrics.py b/src/allmydata/test/test_openmetrics.py index cd5c5d407..c7e26213a 100644 --- a/src/allmydata/test/test_openmetrics.py +++ b/src/allmydata/test/test_openmetrics.py @@ -5,7 +5,95 @@ from allmydata.web.status import Statistics class FakeStatsProvider(object): def get_stats(self): - stats = {'stats': {}, 'counters': {}} + # Parsed into a dict from a running tahoe's /statistics?t=json + stats = {'stats': { + 'storage_server.latencies.get.99_9_percentile': None, + 'storage_server.latencies.close.10_0_percentile': 0.00021910667419433594, + 'storage_server.latencies.read.01_0_percentile': 2.8848648071289062e-05, + 'storage_server.latencies.writev.99_9_percentile': None, + 'storage_server.latencies.read.99_9_percentile': None, + 'storage_server.latencies.allocate.99_0_percentile': 0.000988006591796875, + 'storage_server.latencies.writev.mean': 0.00045332245070571654, + 'storage_server.latencies.close.99_9_percentile': None, + 'cpu_monitor.15min_avg': 0.00017592000079223033, + 'storage_server.disk_free_for_root': 103289454592, + 'storage_server.latencies.get.99_0_percentile': 0.000347137451171875, + 'storage_server.latencies.get.mean': 0.00021158285060171353, + 'storage_server.latencies.read.90_0_percentile': 8.893013000488281e-05, + 'storage_server.latencies.write.01_0_percentile': 3.600120544433594e-05, + 'storage_server.latencies.write.99_9_percentile': 0.00017690658569335938, + 'storage_server.latencies.close.90_0_percentile': 0.00033211708068847656, + 'storage_server.disk_total': 103497859072, + 'storage_server.latencies.close.95_0_percentile': 0.0003509521484375, + 'storage_server.latencies.readv.samplesize': 1000, + 'storage_server.disk_free_for_nonroot': 103289454592, + 'storage_server.latencies.close.mean': 0.0002715024480059103, + 'storage_server.latencies.writev.95_0_percentile': 0.0007410049438476562, + 'storage_server.latencies.readv.90_0_percentile': 0.0003781318664550781, + 'storage_server.latencies.readv.99_0_percentile': 0.0004050731658935547, + 'storage_server.latencies.allocate.mean': 0.0007128627429454784, + 'storage_server.latencies.close.samplesize': 326, + 'storage_server.latencies.get.50_0_percentile': 0.0001819133758544922, + 'storage_server.latencies.write.50_0_percentile': 4.482269287109375e-05, + 'storage_server.latencies.readv.01_0_percentile': 0.0002970695495605469, + 'storage_server.latencies.get.10_0_percentile': 0.00015687942504882812, + 'storage_server.latencies.allocate.90_0_percentile': 0.0008189678192138672, + 'storage_server.latencies.get.samplesize': 472, + 'storage_server.total_bucket_count': 393, + 'storage_server.latencies.read.mean': 5.936201880959903e-05, + 'storage_server.latencies.allocate.01_0_percentile': 0.0004208087921142578, + 'storage_server.latencies.allocate.99_9_percentile': None, + 'storage_server.latencies.readv.mean': 0.00034061360359191893, + 'storage_server.disk_used': 208404480, + 'storage_server.latencies.allocate.50_0_percentile': 0.0007410049438476562, + 'storage_server.latencies.read.99_0_percentile': 0.00011992454528808594, + 'node.uptime': 3805759.8545179367, + 'storage_server.latencies.writev.10_0_percentile': 0.00035190582275390625, + 'storage_server.latencies.writev.90_0_percentile': 0.0006821155548095703, + 'storage_server.latencies.close.01_0_percentile': 0.00021505355834960938, + 'storage_server.latencies.close.50_0_percentile': 0.0002579689025878906, + 'cpu_monitor.1min_avg': 0.0002130000000003444, + 'storage_server.latencies.writev.50_0_percentile': 0.0004138946533203125, + 'storage_server.latencies.read.95_0_percentile': 9.107589721679688e-05, + 'storage_server.latencies.readv.95_0_percentile': 0.0003859996795654297, + 'storage_server.latencies.write.10_0_percentile': 3.719329833984375e-05, + 'storage_server.accepting_immutable_shares': 1, + 'storage_server.latencies.writev.samplesize': 309, + 'storage_server.latencies.get.95_0_percentile': 0.0003190040588378906, + 'storage_server.latencies.readv.10_0_percentile': 0.00032210350036621094, + 'storage_server.latencies.get.90_0_percentile': 0.0002999305725097656, + 'storage_server.latencies.get.01_0_percentile': 0.0001239776611328125, + 'cpu_monitor.total': 641.4941180000001, + 'storage_server.latencies.write.samplesize': 1000, + 'storage_server.latencies.write.95_0_percentile': 9.489059448242188e-05, + 'storage_server.latencies.read.50_0_percentile': 6.890296936035156e-05, + 'storage_server.latencies.writev.01_0_percentile': 0.00033211708068847656, + 'storage_server.latencies.read.10_0_percentile': 3.0994415283203125e-05, + 'storage_server.latencies.allocate.10_0_percentile': 0.0004949569702148438, + 'storage_server.reserved_space': 0, + 'storage_server.disk_avail': 103289454592, + 'storage_server.latencies.write.99_0_percentile': 0.00011301040649414062, + 'storage_server.latencies.write.90_0_percentile': 9.083747863769531e-05, + 'cpu_monitor.5min_avg': 0.0002370666691157502, + 'storage_server.latencies.write.mean': 5.8008909225463864e-05, + 'storage_server.latencies.readv.50_0_percentile': 0.00033020973205566406, + 'storage_server.latencies.close.99_0_percentile': 0.0004038810729980469, + 'storage_server.allocated': 0, + 'storage_server.latencies.writev.99_0_percentile': 0.0007710456848144531, + 'storage_server.latencies.readv.99_9_percentile': 0.0004780292510986328, + 'storage_server.latencies.read.samplesize': 170, + 'storage_server.latencies.allocate.samplesize': 406, + 'storage_server.latencies.allocate.95_0_percentile': 0.0008411407470703125 + }, 'counters': { + 'storage_server.writev': 309, + 'storage_server.bytes_added': 197836146, + 'storage_server.close': 326, + 'storage_server.readv': 14299, + 'storage_server.allocate': 406, + 'storage_server.read': 170, + 'storage_server.write': 3775, + 'storage_server.get': 472} + } return stats class OpenMetrics(unittest.TestCase): @@ -25,5 +113,3 @@ class OpenMetrics(unittest.TestCase): # The parser throws if it can't parse. # Wrap in a list() to drain the generator. families = list(parser.text_string_to_metric_families(metrics)) - # TODO add more realistic stats, incl. missing (None) values - From 339e1747e7b2636a49c4155f9786e79fdb37de3f Mon Sep 17 00:00:00 2001 From: Florian Sesser Date: Fri, 10 Sep 2021 13:15:56 +0000 Subject: [PATCH 159/269] clean up --- src/allmydata/test/test_openmetrics.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/allmydata/test/test_openmetrics.py b/src/allmydata/test/test_openmetrics.py index c7e26213a..9ef7c9c8e 100644 --- a/src/allmydata/test/test_openmetrics.py +++ b/src/allmydata/test/test_openmetrics.py @@ -108,8 +108,10 @@ class OpenMetrics(unittest.TestCase): metrics = Statistics.render_OPENMETRICS(stats, req) # "The content type MUST be..." - req.setHeader.assert_called_with("content-type", "application/openmetrics-text; version=1.0.0; charset=utf-8") + req.setHeader.assert_called_with( + "content-type", "application/openmetrics-text; version=1.0.0; charset=utf-8" + ) - # The parser throws if it can't parse. - # Wrap in a list() to drain the generator. + # The parser throws if it does not like its input. + # Wrapped in a list() to drain the generator. families = list(parser.text_string_to_metric_families(metrics)) From ad84f5df2b9310df2fac25c5dc989c4044af491b Mon Sep 17 00:00:00 2001 From: Florian Sesser Date: Fri, 10 Sep 2021 13:21:06 +0000 Subject: [PATCH 160/269] newline at the end. --- src/allmydata/test/test_openmetrics.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/allmydata/test/test_openmetrics.py b/src/allmydata/test/test_openmetrics.py index 9ef7c9c8e..fdb645b42 100644 --- a/src/allmydata/test/test_openmetrics.py +++ b/src/allmydata/test/test_openmetrics.py @@ -115,3 +115,4 @@ class OpenMetrics(unittest.TestCase): # The parser throws if it does not like its input. # Wrapped in a list() to drain the generator. families = list(parser.text_string_to_metric_families(metrics)) + From 570f15284ab77c8314b4181b9ef8de52d54375fa Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 10 Sep 2021 09:44:49 -0400 Subject: [PATCH 161/269] More tests for IStorageServer.get_buckets(). --- newsfragments/3795.minor | 0 src/allmydata/test/test_istorageserver.py | 88 +++++++++++++++++++++++ 2 files changed, 88 insertions(+) create mode 100644 newsfragments/3795.minor diff --git a/newsfragments/3795.minor b/newsfragments/3795.minor new file mode 100644 index 000000000..e69de29bb diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index 75fbe42e2..74d4fb033 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -255,6 +255,94 @@ class IStorageServerImmutableAPIsTestsMixin(object): (yield buckets[3].callRemote("read", 0, 1024)), b"5" * 24 + b"6" * 1000 ) + @inlineCallbacks + def test_get_buckets_skips_unfinished_buckets(self): + """ + Buckets that are not fully written are not returned by + ``IStorageServer.get_buckets()`` implementations. + """ + si = new_storage_index() + (_, allocated) = yield self.storage_server.allocate_buckets( + si, + new_secret(), + new_secret(), + set(range(5)), + 10, + Referenceable(), + ) + + # Bucket 1 is fully written + yield allocated[1].callRemote("write", 0, b"1" * 10) + yield allocated[1].callRemote("close") + + # Bucket 2 is partially written + yield allocated[2].callRemote("write", 0, b"1" * 5) + + buckets = yield self.storage_server.get_buckets(si) + self.assertEqual(set(buckets.keys()), {1}) + + @inlineCallbacks + def test_read_bucket_at_offset(self): + """ + Given a read bucket returned from ``IStorageServer.get_buckets()``, it + is possible to read at different offsets and lengths, with reads past + the end resulting in empty bytes. + """ + length = 256 * 17 + + si = new_storage_index() + (_, allocated) = yield self.storage_server.allocate_buckets( + si, + new_secret(), + new_secret(), + set(range(1)), + length, + Referenceable(), + ) + + total_data = _randbytes(256) * 17 + yield allocated[0].callRemote("write", 0, total_data) + yield allocated[0].callRemote("close") + + buckets = yield self.storage_server.get_buckets(si) + bucket = buckets[0] + for start, to_read in [ + (0, 250), # fraction + (0, length), # whole thing + (100, 1024), # offset fraction + (length + 1, 100), # completely out of bounds + (length - 100, 200), # partially out of bounds + ]: + data = yield bucket.callRemote("read", start, to_read) + self.assertEqual( + data, + total_data[start : start + to_read], + "Didn't match for start {}, length {}".format(start, to_read), + ) + + @inlineCallbacks + def test_bucket_advise_corrupt_share(self): + """ + Calling ``advise_corrupt_share()`` on a bucket returned by + ``IStorageServer.get_buckets()`` does not result in error (other + behavior is opaque at this level of abstraction). + """ + si = new_storage_index() + (_, allocated) = yield self.storage_server.allocate_buckets( + si, + new_secret(), + new_secret(), + set(range(1)), + 10, + Referenceable(), + ) + + yield allocated[0].callRemote("write", 0, b"0123456789") + yield allocated[0].callRemote("close") + + buckets = yield self.storage_server.get_buckets(si) + yield buckets[0].callRemote("advise_corrupt_share", b"OH NO") + class _FoolscapMixin(SystemTestMixin): """Run tests on Foolscap version of ``IStorageServer.""" From 02ad7b9709328319ef568b065de878d8bb9f6cc7 Mon Sep 17 00:00:00 2001 From: "F. E Noel Nfebe" Date: Fri, 10 Sep 2021 15:35:33 +0100 Subject: [PATCH 162/269] Improved python 3 support text format. Co-authored-by: Jean-Paul Calderone --- README.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 3686a58b8..df171b99f 100644 --- a/README.rst +++ b/README.rst @@ -59,7 +59,9 @@ Once tahoe --version works, see `docs/running.rst `__ to learn 🐍 Python 3 Support -------------------- -Python 3 support has been introduced starting with Tahoe-LAFS 1.16.0, alongside Python 2. System admnistrators are advised to start running Tahoe on Python 3 and should expect Python 2 support to be dropped in a future version. Please, feel free to file issues if you run into bugs while running Tahoe on Python 3. +Python 3 support has been introduced starting with Tahoe-LAFS 1.16.0, alongside Python 2. +System administrators are advised to start running Tahoe on Python 3 and should expect Python 2 support to be dropped in a future version. +Please, feel free to file issues if you run into bugs while running Tahoe on Python 3. 🤖 Issues From 38fcbd56e0c1fffe3c4baf0789ce5e855d200dba Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 10 Sep 2021 10:46:17 -0400 Subject: [PATCH 163/269] Adjust the news fragment to match the writing style guide --- newsfragments/3749.documentation | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/newsfragments/3749.documentation b/newsfragments/3749.documentation index 82e1e7856..554564a0b 100644 --- a/newsfragments/3749.documentation +++ b/newsfragments/3749.documentation @@ -1 +1 @@ -Fixes links to the installation and about Tahoe pages in the docs. +Documentation and installation links in the README have been fixed. From ac9875da75c1dd42080f1d18a506a17846671df6 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 10 Sep 2021 11:39:48 -0400 Subject: [PATCH 164/269] Add explanation to new error logging. --- src/allmydata/immutable/downloader/share.py | 2 +- src/allmydata/immutable/layout.py | 2 +- src/allmydata/storage_client.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/allmydata/immutable/downloader/share.py b/src/allmydata/immutable/downloader/share.py index 1e751500b..41e11426f 100644 --- a/src/allmydata/immutable/downloader/share.py +++ b/src/allmydata/immutable/downloader/share.py @@ -477,7 +477,7 @@ class Share(object): str(f.value)) self._rref.callRemote( "advise_corrupt_share", reason.encode("utf-8") - ).addErrback(log.err) + ).addErrback(log.err, "Error from remote call to advise_corrupt_share") def _satisfy_block_hash_tree(self, needed_hashes): o_bh = self.actual_offsets["block_hashes"] diff --git a/src/allmydata/immutable/layout.py b/src/allmydata/immutable/layout.py index 886a5db73..6c7362b8a 100644 --- a/src/allmydata/immutable/layout.py +++ b/src/allmydata/immutable/layout.py @@ -254,7 +254,7 @@ class WriteBucketProxy(object): return d def abort(self): - self._rref.callRemote("abort").addErrback(log.err) + self._rref.callRemote("abort").addErrback(log.err, "Error from remote call to abort an immutable write bucket") def get_servername(self): return self._server.get_name() diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index c278abce3..e9dc8c84c 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -1015,4 +1015,4 @@ class _StorageServer(object): storage_index, shnum, reason, - ).addErrback(log.err) + ).addErrback(log.err, "Error from remote call to advise_corrupt_share") From fb61e29b5933fb9f84df4fc319e48a56f8e22659 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 10 Sep 2021 11:40:30 -0400 Subject: [PATCH 165/269] Better explanation. --- newsfragments/3779.bugfix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/newsfragments/3779.bugfix b/newsfragments/3779.bugfix index c6745f1af..073046474 100644 --- a/newsfragments/3779.bugfix +++ b/newsfragments/3779.bugfix @@ -1 +1 @@ -Fixed bug where share corruption events were not recorded on Windows. \ No newline at end of file +Fixed bug where share corruption events were not logged on storage servers running on Windows. \ No newline at end of file From b0d635c9c9537d625112348228aa3a084f43a598 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 13 Sep 2021 09:15:20 -0400 Subject: [PATCH 166/269] Use skip decorator. --- src/allmydata/test/test_istorageserver.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index 75fbe42e2..43ddd09ce 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -19,6 +19,7 @@ if PY2: # fmt: on from random import randrange +from unittest import skipIf from twisted.internet.defer import inlineCallbacks from twisted.trial.unittest import TestCase @@ -86,6 +87,7 @@ class IStorageServerImmutableAPIsTestsMixin(object): self.assertEqual(set(allocated.keys()), set(range(5))) # We validate the bucket objects' interface in a later test. + @skipIf(True, "https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3793") @inlineCallbacks def test_allocate_buckets_repeat(self): """ @@ -118,10 +120,7 @@ class IStorageServerImmutableAPIsTestsMixin(object): self.assertEqual(already_got, already_got2) self.assertEqual(set(allocated.keys()), set(allocated2.keys())) - test_allocate_buckets_repeat.skip = ( - "https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3793" - ) - + @skipIf(True, "https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3793") @inlineCallbacks def test_allocate_buckets_more_sharenums(self): """ @@ -154,10 +153,6 @@ class IStorageServerImmutableAPIsTestsMixin(object): self.assertEqual(already_got2, set()) # none were fully written self.assertEqual(set(allocated2.keys()), set(range(7))) - test_allocate_buckets_more_sharenums.skip = ( - "https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3793" - ) - @inlineCallbacks def test_written_shares_are_allocated(self): """ From 1fa0f72c8e217fa5e3131cf84556ba1b70198b74 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 13 Sep 2021 09:28:03 -0400 Subject: [PATCH 167/269] Use AsyncTestCase. --- src/allmydata/test/test_istorageserver.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index 43ddd09ce..9bf7e6687 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -19,15 +19,16 @@ if PY2: # fmt: on from random import randrange -from unittest import skipIf + +from testtools import skipIf from twisted.internet.defer import inlineCallbacks -from twisted.trial.unittest import TestCase from foolscap.api import Referenceable from allmydata.interfaces import IStorageServer from .test_system import SystemTestMixin +from .common import AsyncTestCase def _randbytes(length): @@ -87,8 +88,8 @@ class IStorageServerImmutableAPIsTestsMixin(object): self.assertEqual(set(allocated.keys()), set(range(5))) # We validate the bucket objects' interface in a later test. - @skipIf(True, "https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3793") @inlineCallbacks + @skipIf(True, "https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3793") def test_allocate_buckets_repeat(self): """ allocate_buckets() with the same storage index returns the same result, @@ -256,6 +257,7 @@ class _FoolscapMixin(SystemTestMixin): @inlineCallbacks def setUp(self): + AsyncTestCase.setUp(self) self.basedir = "test_istorageserver/" + self.id() yield SystemTestMixin.setUp(self) yield self.set_up_nodes(1) @@ -264,14 +266,19 @@ class _FoolscapMixin(SystemTestMixin): ).get_storage_server() self.assertTrue(IStorageServer.providedBy(self.storage_server)) + @inlineCallbacks + def tearDown(self): + AsyncTestCase.tearDown(self) + yield SystemTestMixin.tearDown(self) + class FoolscapSharedAPIsTests( - _FoolscapMixin, IStorageServerSharedAPIsTestsMixin, TestCase + _FoolscapMixin, IStorageServerSharedAPIsTestsMixin, AsyncTestCase ): """Foolscap-specific tests for shared ``IStorageServer`` APIs.""" class FoolscapImmutableAPIsTests( - _FoolscapMixin, IStorageServerImmutableAPIsTestsMixin, TestCase + _FoolscapMixin, IStorageServerImmutableAPIsTestsMixin, AsyncTestCase ): """Foolscap-specific tests for immutable ``IStorageServer`` APIs.""" From b01c5c7454a5e7cbe0b1299291f29d92fdb59feb Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 13 Sep 2021 09:35:52 -0400 Subject: [PATCH 168/269] Clarify argument names. --- src/allmydata/test/test_istorageserver.py | 70 +++++++++++------------ 1 file changed, 35 insertions(+), 35 deletions(-) diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index 9bf7e6687..8c3a30afa 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -78,11 +78,11 @@ class IStorageServerImmutableAPIsTestsMixin(object): """ (already_got, allocated) = yield self.storage_server.allocate_buckets( new_storage_index(), - new_secret(), - new_secret(), - set(range(5)), - 1024, - Referenceable(), + renew_secret=new_secret(), + cancel_secret=new_secret(), + sharenums=set(range(5)), + allocated_size=1024, + canary=Referenceable(), ) self.assertEqual(already_got, set()) self.assertEqual(set(allocated.keys()), set(range(5))) @@ -97,21 +97,21 @@ class IStorageServerImmutableAPIsTestsMixin(object): This fails due to https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3793 """ - si, renew_secret, cancel_secret = ( + storage_index, renew_secret, cancel_secret = ( new_storage_index(), new_secret(), new_secret(), ) (already_got, allocated) = yield self.storage_server.allocate_buckets( - si, + storage_index, renew_secret, cancel_secret, - set(range(5)), - 1024, - Referenceable(), + sharenums=set(range(5)), + allocated_size=1024, + canary=Referenceable(), ) (already_got2, allocated2) = yield self.storage_server.allocate_buckets( - si, + storage_index, renew_secret, cancel_secret, set(range(5)), @@ -130,26 +130,26 @@ class IStorageServerImmutableAPIsTestsMixin(object): Fails due to https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3793 """ - si, renew_secret, cancel_secret = ( + storage_index, renew_secret, cancel_secret = ( new_storage_index(), new_secret(), new_secret(), ) yield self.storage_server.allocate_buckets( - si, + storage_index, renew_secret, cancel_secret, - set(range(5)), - 1024, - Referenceable(), + sharenums=set(range(5)), + allocated_size=1024, + canary=Referenceable(), ) (already_got2, allocated2) = yield self.storage_server.allocate_buckets( - si, + storage_index, renew_secret, cancel_secret, - set(range(7)), - 1024, - Referenceable(), + sharenums=set(range(7)), + allocated_size=1024, + canary=Referenceable(), ) self.assertEqual(already_got2, set()) # none were fully written self.assertEqual(set(allocated2.keys()), set(range(7))) @@ -161,18 +161,18 @@ class IStorageServerImmutableAPIsTestsMixin(object): ``IStoragServer.allocate_buckets()``. Partially-written or empty shares don't. """ - si, renew_secret, cancel_secret = ( + storage_index, renew_secret, cancel_secret = ( new_storage_index(), new_secret(), new_secret(), ) (_, allocated) = yield self.storage_server.allocate_buckets( - si, + storage_index, renew_secret, cancel_secret, - set(range(5)), - 1024, - Referenceable(), + sharenums=set(range(5)), + allocated_size=1024, + canary=Referenceable(), ) # Bucket 1 is fully written in one go. @@ -188,12 +188,12 @@ class IStorageServerImmutableAPIsTestsMixin(object): yield allocated[0].callRemote("write", 0, b"1" * 512) (already_got, _) = yield self.storage_server.allocate_buckets( - si, + storage_index, renew_secret, cancel_secret, - set(range(5)), - 1024, - Referenceable(), + sharenums=set(range(5)), + allocated_size=1024, + canary=Referenceable(), ) self.assertEqual(already_got, {1, 2}) @@ -208,18 +208,18 @@ class IStorageServerImmutableAPIsTestsMixin(object): 2. When overlapping writes happen, the resulting read returns the earliest written value. """ - si, renew_secret, cancel_secret = ( + storage_index, renew_secret, cancel_secret = ( new_storage_index(), new_secret(), new_secret(), ) (_, allocated) = yield self.storage_server.allocate_buckets( - si, + storage_index, renew_secret, cancel_secret, - set(range(5)), - 1024, - Referenceable(), + sharenums=set(range(5)), + allocated_size=1024, + canary=Referenceable(), ) # Bucket 1 is fully written in order @@ -238,7 +238,7 @@ class IStorageServerImmutableAPIsTestsMixin(object): yield allocated[3].callRemote("write", 24, b"6" * 1000) yield allocated[3].callRemote("close") - buckets = yield self.storage_server.get_buckets(si) + buckets = yield self.storage_server.get_buckets(storage_index) self.assertEqual(set(buckets.keys()), {1, 2, 3}) self.assertEqual( From a482f216a1353ad9d3829f917702172c81d43258 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 13 Sep 2021 09:40:32 -0400 Subject: [PATCH 169/269] Use more reproducible "random" numbers. --- src/allmydata/test/test_istorageserver.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index 8c3a30afa..ba6f240d7 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -18,7 +18,7 @@ if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 # fmt: on -from random import randrange +from random import Random from testtools import skipIf @@ -31,10 +31,15 @@ from .test_system import SystemTestMixin from .common import AsyncTestCase +# Use random generator with known seed, so results are reproducible if tests +# are run in the same order. +_RANDOM = Random(0) + + def _randbytes(length): # type: (int) -> bytes """Return random bytes string of given length.""" - return b"".join([bchr(randrange(0, 256)) for _ in range(length)]) + return b"".join([bchr(_RANDOM.randrange(0, 256)) for _ in range(length)]) def new_storage_index(): From d04cd13b37fb2c4fa21154e24a23f8627bbad47f Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 13 Sep 2021 09:43:03 -0400 Subject: [PATCH 170/269] Actual test for get_version(). --- src/allmydata/test/test_istorageserver.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index ba6f240d7..9d164c021 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -63,9 +63,13 @@ class IStorageServerSharedAPIsTestsMixin(object): @inlineCallbacks def test_version(self): - # TODO get_version() returns a dict-like thing with some of the - # expected fields. - yield self.storage_server.get_version() + """ + ``IStorageServer`` returns a dictionary where the key is an expected + protocol version. + """ + result = yield self.storage_server.get_version() + self.assertIsInstance(result, dict) + self.assertIn(b"http://allmydata.org/tahoe/protocols/storage/v1", result) class IStorageServerImmutableAPIsTestsMixin(object): From e96c22914bd040cd89d975d1c33e0e3ecdc09c8e Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 13 Sep 2021 09:43:53 -0400 Subject: [PATCH 171/269] Fix typo. --- src/allmydata/test/test_istorageserver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index 9d164c021..50e803920 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -167,7 +167,7 @@ class IStorageServerImmutableAPIsTestsMixin(object): def test_written_shares_are_allocated(self): """ Shares that are fully written to show up as allocated in result from - ``IStoragServer.allocate_buckets()``. Partially-written or empty + ``IStorageServer.allocate_buckets()``. Partially-written or empty shares don't. """ storage_index, renew_secret, cancel_secret = ( From 86ea3ca4084f723ea870ca424c7b5335e59e78ba Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 13 Sep 2021 09:49:27 -0400 Subject: [PATCH 172/269] Demonstrate that last write wins. --- src/allmydata/test/test_istorageserver.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index 50e803920..5be5814c4 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -243,8 +243,9 @@ class IStorageServerImmutableAPIsTestsMixin(object): # Bucket 3 has an overlapping write. yield allocated[3].callRemote("write", 0, b"5" * 20) - yield allocated[3].callRemote("write", 0, b"5" * 24) - yield allocated[3].callRemote("write", 24, b"6" * 1000) + # The second write will overwrite the first. + yield allocated[3].callRemote("write", 0, b"6" * 24) + yield allocated[3].callRemote("write", 24, b"7" * 1000) yield allocated[3].callRemote("close") buckets = yield self.storage_server.get_buckets(storage_index) @@ -257,7 +258,8 @@ class IStorageServerImmutableAPIsTestsMixin(object): (yield buckets[2].callRemote("read", 0, 1024)), b"3" * 512 + b"4" * 512 ) self.assertEqual( - (yield buckets[3].callRemote("read", 0, 1024)), b"5" * 24 + b"6" * 1000 + (yield buckets[3].callRemote("read", 0, 1024)), + b"6" * 24 + b"7" * 1000, ) From 977b6065b1fd308902617e60be9faf5e3123e31c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 13 Sep 2021 09:50:38 -0400 Subject: [PATCH 173/269] Match actual behavior. --- src/allmydata/test/test_istorageserver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index 5be5814c4..902045bdb 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -215,7 +215,7 @@ class IStorageServerImmutableAPIsTestsMixin(object): happened, only by their offsets. 2. When overlapping writes happen, the resulting read returns the - earliest written value. + latest written value. """ storage_index, renew_secret, cancel_secret = ( new_storage_index(), From 9ce7cce712350f139c03eacdd223eadf0b977fd8 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 13 Sep 2021 10:10:07 -0400 Subject: [PATCH 174/269] Refactor SystemTestMixin into its own module. --- src/allmydata/test/common_system.py | 910 ++++++++++++++++++++++ src/allmydata/test/test_istorageserver.py | 2 +- src/allmydata/test/test_system.py | 895 +-------------------- 3 files changed, 917 insertions(+), 890 deletions(-) create mode 100644 src/allmydata/test/common_system.py diff --git a/src/allmydata/test/common_system.py b/src/allmydata/test/common_system.py new file mode 100644 index 000000000..6bda4e52d --- /dev/null +++ b/src/allmydata/test/common_system.py @@ -0,0 +1,910 @@ +""" +Test infrastructure for integration-y tests that run actual nodes, like those +in ``allmydata.test.test_system``. + +Ported to Python 3. +""" + +from future.utils import PY2 +if PY2: + # Don't import bytes since it causes issues on (so far unported) modules on Python 2. + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, dict, list, object, range, max, min, str # noqa: F401 + +import os +from functools import partial + +from twisted.internet import reactor +from twisted.internet import defer +from twisted.internet.defer import inlineCallbacks +from twisted.application import service + +from foolscap.api import flushEventualQueue + +from allmydata import client +from allmydata.introducer.server import create_introducer +from allmydata.util import fileutil, log, pollmixin + +from twisted.python.filepath import ( + FilePath, +) + +from .common import ( + TEST_RSA_KEY_SIZE, + SameProcessStreamEndpointAssigner, +) + +from . import common_util as testutil +from ..scripts.common import ( + write_introducer, +) + +# our system test uses the same Tub certificates each time, to avoid the +# overhead of key generation +SYSTEM_TEST_CERTS = [ +"""-----BEGIN CERTIFICATE----- +MIICojCCAYoCAQEwDQYJKoZIhvcNAQELBQAwFzEVMBMGA1UEAwwMbmV3cGJfdGhp +bmd5MB4XDTIwMDEwMjAxNDAzM1oXDTIxMDEwMTAxNDAzM1owFzEVMBMGA1UEAwwM +bmV3cGJfdGhpbmd5MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA1iNV +z07PYwZwucl87QlL2TFZvDxD4flZ/p3BZE3DCT5Efn9w2NT4sHXL1e+R/qsDFuNG +bw1y1TRM0DGK6Wr0XRT2mLQULNgB8y/HrhcSdONsYRyWdj+LimyECKjwh0iSkApv +Yj/7IOuq6dOoh67YXPdf75OHLShm4+8q8fuwhBL+nuuO4NhZDJKupYHcnuCkcF88 +LN77HKrrgbpyVmeghUkwJMLeJCewvYVlambgWRiuGGexFgAm6laS3rWetOcdm9eg +FoA9PKNN6xvPatbj99MPoLpBbzsI64M0yT/wTSw1pj/Nom3rwfMa2OH8Kk7c8R/r +U3xj4ZY1DTlGERvejQIDAQABMA0GCSqGSIb3DQEBCwUAA4IBAQAwyQjQ3ZgtJ3JW +r3/EPdqSUBamTfXIpOh9rXmRjPpbe+MvenqIzl4q+GnkL5mdEb1e1hdKQZgFQ5Q5 +tbcNIz6h5C07KaNtbqhZCx5c/RUEH87VeXuAuOqZHbZWJ18q0tnk+YgWER2TOkgE +RI2AslcsJBt88UUOjHX6/7J3KjPFaAjW1QV3TTsHxk14aYDYJwPdz+ijchgbOPQ0 +i+ilhzcB+qQnOC1s4xQSFo+zblTO7EgqM9KpupYfOVFh46P1Mak2W8EDvhz0livl +OROXJ6nR/13lmQdfVX6T45d+ITBwtmW2nGAh3oI3JlArGKHaW+7qnuHR72q9FSES +cEYA/wmk +-----END CERTIFICATE----- +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDWI1XPTs9jBnC5 +yXztCUvZMVm8PEPh+Vn+ncFkTcMJPkR+f3DY1PiwdcvV75H+qwMW40ZvDXLVNEzQ +MYrpavRdFPaYtBQs2AHzL8euFxJ042xhHJZ2P4uKbIQIqPCHSJKQCm9iP/sg66rp +06iHrthc91/vk4ctKGbj7yrx+7CEEv6e647g2FkMkq6lgdye4KRwXzws3vscquuB +unJWZ6CFSTAkwt4kJ7C9hWVqZuBZGK4YZ7EWACbqVpLetZ605x2b16AWgD08o03r +G89q1uP30w+gukFvOwjrgzTJP/BNLDWmP82ibevB8xrY4fwqTtzxH+tTfGPhljUN +OUYRG96NAgMBAAECggEAJ5xztBx0+nFnisZ9yG8uy6d4XPyc5gE1J4dRDdfgmyYc +j3XNjx6ePi4cHZ/qVryVnrc+AS7wrgW1q9FuS81QFKPbFdZB4SW3/p85BbgY3uxu +0Ovz3T3V9y4polx12eCP0/tKLVd+gdF2VTik9Sxfs5rC8VNN7wmJNuK4A/k15sgy +BIu/R8NlMNGQySNhtccp+dzB8uTyKx5zFZhVvnAK/3YX9BC2V4QBW9JxO4S8N0/9 +48e9Sw/fGCfQ/EFPKGCvTvfuRqJ+4t5k10FygXJ+s+y70ifYi+aSsjJBuranbLJp +g5TwhuKnTWs8Nth3YRLbcJL4VBIOehjAWy8pDMMtlQKBgQD0O8cHb8cOTGW0BijC +NDofhA2GooQUUR3WL324PXWZq0DXuBDQhJVBKWO3AYonivhhd/qWO8lea9MEmU41 +nKZ7maS4B8AJLJC08P8GL1uCIE/ezEXEi9JwC1zJiyl595Ap4lSAozH0DwjNvmGL +5mIdYg0BliqFXbloNJkNlb7INwKBgQDgdGEIWXc5Y1ncWNs6iDIV/t2MlL8vLrP0 +hpkl/QiMndOQyD6JBo0+ZqvOQTSS4NTSxBROjPxvFbEJ3eH8Pmn8gHOf46fzP1OJ +wlYv0gYzkN4FE/tN6JnO2u9pN0euyyZLM1fnEcrMWColMN8JlWjtA7Gbxm8lkfa4 +3vicaJtlWwKBgQCQYL4ZgVR0+Wit8W4qz+EEPHYafvwBXqp6sXxqa7qXawtb+q3F +9nqdGLCfwMNA+QA37ksugI1byfXmpBH902r/aiZbvAkj4zpwHH9F0r0PwbY1iSA9 +PkLahX0Gj8OnHFgWynsVyGOBWVnk9oSHxVt+7zWtGG5uhKdUGLPZugocJQKBgB61 +7bzduOFiRZ5PjhdxISE/UQL2Kz6Cbl7rt7Kp72yF/7eUnnHTMqoyFBnRdCcQmi4I +ZBrnUXbFigamlFAWHhxNWwSqeoVeychUjcRXQT/291nMhRsA02KpNA66YJV6+E9b +xBA6r/vLqGCUUkAWcFfVpIyC1xxV32MmJvAHpBN3AoGAPF3MUFiO0iKNZfst6Tm3 +rzrldLawDo98DRZ7Yb2kWlWZYqUk/Nvryvo2cns75WGSMDYVbbRp+BY7kZmNYa9K +iQzKDL54ZRu6V+getJdeAO8yXoCmnZKxt5OHvOSrQMfAmFKSwLwxBbZBfXEyuune +yfusXLtCgajpreoVIa0xWdQ= +-----END PRIVATE KEY----- +""", # 0 +"""-----BEGIN CERTIFICATE----- +MIICojCCAYoCAQEwDQYJKoZIhvcNAQELBQAwFzEVMBMGA1UEAwwMbmV3cGJfdGhp +bmd5MB4XDTIwMDEwMjAxNDAzM1oXDTIxMDEwMTAxNDAzM1owFzEVMBMGA1UEAwwM +bmV3cGJfdGhpbmd5MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEApDzW +4ZBeK9w4xpRaed6lXzeCO0Xmr3f0ynbueSdiZ89FWoAMgK+SiBIOViYV6hfm0Wah +lemSNzFGx5LvDSg2uwSqEP23DeM9O/SQPgIAiLeeEsYZJcgg2jz92YfFEaahsGdI +6qSP4XI2/5dgKRpPOYDGyw6R5PQR6w22Xq1WD1jBvImk/k09I9jHRn40pYbaJzbg +U2aIjvOruo2kqe4f6iDqE0piYimAZJUvemu1UoyV5NG590hGkDuWsMD77+d2FxCj +9Nzb+iuuG3ksnanHPyXi1hQmzp5OmzVWaevCHinNjWgsuSuLGO9H2SLf3wwp2UCs +EpKtzoKrnZdEg/anNwIDAQABMA0GCSqGSIb3DQEBCwUAA4IBAQChxtr67o1aZZMJ +A6gESPtFjZLw6wG0j50JsrWKLvoXVts1ToJ9u2nx01aFKjBwb4Yg+vdJfDgIIAEm +jS56h6H2DfJlkTWHmi8Vx1wuusWnrNwYMI53tdlRIpD2+Ne7yeoLQZcVN2wuPmxD +Mbksg4AI4csmbkU/NPX5DtMy4EzM/pFvIcxNIVRUMVTFzn5zxhKfhyPqrMI4fxw1 +UhUbEKO+QgIqTNp/dZ0lTbFs5HJQn6yirWyyvQKBPmaaK+pKd0RST/T38OU2oJ/J +LojRs7ugCJ+bxJqegmQrdcVqZZGbpYeK4O/5eIn8KOlgh0nUza1MyjJJemgBBWf7 +HoXB8Fge +-----END CERTIFICATE----- +-----BEGIN PRIVATE KEY----- +MIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQCkPNbhkF4r3DjG +lFp53qVfN4I7Reavd/TKdu55J2Jnz0VagAyAr5KIEg5WJhXqF+bRZqGV6ZI3MUbH +ku8NKDa7BKoQ/bcN4z079JA+AgCIt54SxhklyCDaPP3Zh8URpqGwZ0jqpI/hcjb/ +l2ApGk85gMbLDpHk9BHrDbZerVYPWMG8iaT+TT0j2MdGfjSlhtonNuBTZoiO86u6 +jaSp7h/qIOoTSmJiKYBklS96a7VSjJXk0bn3SEaQO5awwPvv53YXEKP03Nv6K64b +eSydqcc/JeLWFCbOnk6bNVZp68IeKc2NaCy5K4sY70fZIt/fDCnZQKwSkq3Ogqud +l0SD9qc3AgMBAAECggEBAIu55uaIOFYASZ1IYaEFNpRHWVisI5Js76nAfSo9w46l +3E8eWYSx2mxBUEkipco/A3RraFVuHaMvHRR1gUMkT0vUsAs8jxwVk+cKLh1S/rlR +3f4C4yotlSWWdjE3PQXDShQWCwb1ciNPVFMmqfzOEVDOqlHe12h97TCYverWdT0f +3LZICLQsZd1WPKnPNXrsRRDCBuRLapdg+M0oJ+y6IiCdm+qM7Qvaoef6hlvm5ECz +LCM92db5BKTuPOQXMx2J8mjaBgU3aHxRV08IFgs7mI6q0t0FM7LlytIAJq1Hg5QU +36zDKo8tblkPijWZWlqlZCnlarrd3Ar/BiLEiuOGDMECgYEA1GOp0KHy0mbJeN13 ++TDsgP7zhmqFcuJREu2xziNJPK2S06NfGYE8vuVqBGzBroLTQ3dK7rOJs9C6IjCE +mH7ZeHzfcKohpZnl443vHMSpgdh/bXTEO1aQZNbJ2hLYs8ie/VqqHR0u6YtpUqZL +LgaUA0U8GnlsO55B8kyCelckmDkCgYEAxfYQMPEEzg1tg2neqEfyoeY0qQTEJTeh +CPMztowSJpIyF1rQH6TaG0ZchkiAkw3W58RVDfvK72TuVlC5Kz00C2/uPnrqm0dX +iMPeML5rFlG3VGCrSTnAPI+az6P65q8zodqcTtA8xoxgPOlc/lINOxiTEMxLyeGF +8GyP+sCM2u8CgYEAvMBR05OJnEky9hJEpBZBqSZrQGL8dCwDh0HtCdi8JovPd/yx +8JW1aaWywXnx6uhjXoru8hJm54IxWV8rB+d716OKY7MfMfACqWejQDratgW0wY7L +MjztGGD2hLLJGYXLHjfsBPHBllaKZKRbHe1Er19hWdndQWKVEwPB1X4KjKkCgYEA +nWHmN3K2djbYtRyLR1CEBtDlVuaSJmCWp23q1BuCJqYeKtEpG69NM1f6IUws5Dyh +eXtuf4KKMU8V6QueW1D6OomPaJ8CO9c5MWM/F5ObwY/P58Y/ByVhvwQQeToONC5g +JzKNCF+nodZigKqrIwoKuMvtx/IT4vloKd+1jA5fLYMCgYBoT3HLCyATVdDSt1TZ +SbEDoLSYt23KRjQV93+INP949dYCagtgh/kTzxBopw5FljISLfdYizIRo2AzhhfP +WWpILlnt19kD+sNirJVqxJacfEZsu5baWTedI/yrCuVsAs/s3/EEY6q0Qywknxtp +Fwh1/8y5t14ib5fxOVhi8X1nEA== +-----END PRIVATE KEY----- +""", # 1 +"""-----BEGIN CERTIFICATE----- +MIICojCCAYoCAQEwDQYJKoZIhvcNAQELBQAwFzEVMBMGA1UEAwwMbmV3cGJfdGhp +bmd5MB4XDTIwMDEwMjAxNDAzM1oXDTIxMDEwMTAxNDAzM1owFzEVMBMGA1UEAwwM +bmV3cGJfdGhpbmd5MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAwMTn +hXnpKHGAir3WYbOxefVrMA07OZNAsNa29nBwLA+NVIJNUFgquibMj7QYo8+M45oY +6LKr4yRcBryZVvyxfdr92xp8+kLeVApk2WLjkdBTRagHh9qdrY0hQmagCBN6/hLG +Xug8VksQUdhX3vu6ZyMvTLfKRkDOMRVkRGRGg/dOcvom7zpqMCGYenMG2FStr6UV +3s3dlCSZZTdTX5Uoq6yfUUJE3nITGKjpnpJKqIs3PWCIxdj7INIcjJKvIdUcavIV +2hEhh60A8ltmtdpQAXVBE+U7aZgS1fGAWS2A0a3UwuP2pkQp6OyKCUVHpZatbl9F +ahDN2QBzegv/rdJ1zwIDAQABMA0GCSqGSIb3DQEBCwUAA4IBAQAl4OQZ+FB9ZSUv +FL/KwLNt+ONU8Sve/xiX+8vKAvgKm2FrjyK+AZPwibnu+FSt2G4ndZBx4Wvpe5V+ +gCsbzSXlh9cDn2SRXyprt2l/8Fj4eUMaThmLKOK200/N/s2SpmBtnuflBrhNaJpw +DEi2KEPuXsgvkuVzXN06j75cUHwn5LeWDAh0RalkVuGbEWBoFx9Hq8WECdlCy0YS +y09+yO01qz70y88C2rPThKw8kP4bX8aFZbvsnRHsLu/8nEQNlrELcfBarPVHjJ/9 +imxOdymJkV152V58voiXP/PwXhynctQbF7e+0UZ+XEGdbAbZA0BMl7z+b09Z+jF2 +afm4mVox +-----END CERTIFICATE----- +-----BEGIN PRIVATE KEY----- +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDAxOeFeekocYCK +vdZhs7F59WswDTs5k0Cw1rb2cHAsD41Ugk1QWCq6JsyPtBijz4zjmhjosqvjJFwG +vJlW/LF92v3bGnz6Qt5UCmTZYuOR0FNFqAeH2p2tjSFCZqAIE3r+EsZe6DxWSxBR +2Ffe+7pnIy9Mt8pGQM4xFWREZEaD905y+ibvOmowIZh6cwbYVK2vpRXezd2UJJll +N1NflSirrJ9RQkTechMYqOmekkqoizc9YIjF2Psg0hyMkq8h1Rxq8hXaESGHrQDy +W2a12lABdUET5TtpmBLV8YBZLYDRrdTC4/amRCno7IoJRUellq1uX0VqEM3ZAHN6 +C/+t0nXPAgMBAAECggEAF+2ZK4lZdsq4AQDVhqUuh4v+NSW/T0NHCWxto6OLWPzJ +N09BV5LKIvdD9yaM1HCj9XCgXOooyfYuciuhARo20f+H+VWNY+c+/8GWiSFsTCJG +4+Oao7NwVSWqljp07Ou2Hamo9AjxzGhe6znmlmg62CiW63f45MWQkqksHA0yb5jg +/onJ2//I+OI+aTKNfjt1G6h2x7oxeGTU1jJ0Hb2xSh+Mpqx9NDfb/KZyOndhSG5N +xRVosQ6uV+9mqHxTTwTZurTG31uhZzarkMuqxhcHS94ub7berEc/OlqvbyMKNZ3A +lzuvq0NBZhEUhAVgORAIS17r/q2BvyG4u5LFbG2p0QKBgQDeyyOl+A7xc4lPE2OL +Z3KHJPP4RuUnHnWFC+bNdr5Ag8K7jcjZIcasyUom9rOR0Fpuw9wmXpp3+6fyp9bJ +y6Bi5VioR0ZFP5X+nXxIN3yvgypu6AZvkhHrEFer+heGHxPlbwNKCKMbPzDZPBTZ +vlC7g7xUUcpNmGhrOKr3Qq5FlwKBgQDdgCmRvsHUyzicn8TI3IJBAOcaQG0Yr/R2 +FzBqNfHHx7fUZlJfKJsnu9R9VRZmBi4B7MA2xcvz4QrdZWEtY8uoYp8TAGILfW1u +CP4ZHrzfDo/67Uzk2uTMTd0+JOqSm/HiVNguRPvC8EWBoFls+h129GKThMvKR1hP +1oarfAGIiQKBgQCIMAq5gHm59JMhqEt4QqMKo3cS9FtNX1wdGRpbzFMd4q0dstzs +ha4Jnv3Z9YHtBzzQap9fQQMRht6yARDVx8hhy6o3K2J0IBtTSfdXubtZGkfNBb4x +Y0vaseG1uam5jbO+0u5iygbSN/1nPUfNln2JMkzkCh8s8ZYavMgdX0BiPwKBgChR +QL/Hog5yoy5XIoGRKaBdYrNzkKgStwObuvNKOGUt5DckHNA3Wu6DkOzzRO1zKIKv +LlmJ7VLJ3qln36VcaeCPevcBddczkGyb9GxsHOLZCroY4YsykLzjW2cJXy0qd3/E +A8mAQvc7ttsebciZSi2x1BOX82QxUlDN8ptaKglJAoGBAMnLN1TQB0xtWYDPGcGV +2IvgX7OTRRlMVrTvIOvP5Julux9z1r0x0cesl/jaXupsBUlLLicPyBMSBJrXlr24 +mrgkodk4TdqO1VtBCZBqak97DHVezstMrbpCGlUD5jBnsHVRLERvS09QlGhqMeNL +jpNQbWH9VhutzbvpYquKrhvK +-----END PRIVATE KEY----- +""", # 2 +"""-----BEGIN CERTIFICATE----- +MIICojCCAYoCAQEwDQYJKoZIhvcNAQELBQAwFzEVMBMGA1UEAwwMbmV3cGJfdGhp +bmd5MB4XDTIwMDEwMjAxNDAzM1oXDTIxMDEwMTAxNDAzM1owFzEVMBMGA1UEAwwM +bmV3cGJfdGhpbmd5MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAypqi +YTni3s60Uo8vgGcFvjWWkB5CD9Fx9pW/2KcxRJ/u137Y+BG8qWMA4lgII3ZIuvo4 +6rLDiXnAnDZqUtrvZ90O/gH6RyQqX3AI4EwPvCnRIIe0okRcxnxYBL/LfBY54xuv +46JRYZP4c9IImqQH9QVo2/egtEzcpbmT/mfhpf6NGQWC3Xps2BqDT2SV/DrX/wPA +8P1atE1AxNp8ENxK/cjFAteEyDZOsDSa757ZHKAdM7L8rZ1Fd2xAA1Dq7IyYpTNE +IX72xytWxllcNvSUPLT+oicsSZBadc/p3moc3tR/rNdgrHKybedadru/f9Gwpa+v +0sllZlEcVPSYddAzWwIDAQABMA0GCSqGSIb3DQEBCwUAA4IBAQCmk60Nj5FPvemx +DSSQjJPyJoIDpTxQ4luSzIq4hPwlUXw7dqrvHyCWgn2YVe9xZsGrT/+n376ecmgu +sw4s4qVhR9bzKkTMewjC2wUooTA5v9HYsNWZy3Ah7hHPbDHlMADYobjB5/XolNUP +bCM9xALEdM9DxpC4vjUZexlRKmjww9QKE22jIM+bqsK0zqDSq+zHpfHNGGcS3vva +OvI6FPc1fAr3pZpVzevMSN2zufIJwjL4FT5/uzwOCaSCwgR1ztD5CSbQLTLlwIsX +S7h2WF9078XumeRjKejdjEjyH4abKRq8+5LVLcjKEpg7OvktuRpPoGPCEToaAzuv +h+RSQwwY +-----END CERTIFICATE----- +-----BEGIN PRIVATE KEY----- +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDKmqJhOeLezrRS +jy+AZwW+NZaQHkIP0XH2lb/YpzFEn+7Xftj4EbypYwDiWAgjdki6+jjqssOJecCc +NmpS2u9n3Q7+AfpHJCpfcAjgTA+8KdEgh7SiRFzGfFgEv8t8FjnjG6/jolFhk/hz +0giapAf1BWjb96C0TNyluZP+Z+Gl/o0ZBYLdemzYGoNPZJX8Otf/A8Dw/Vq0TUDE +2nwQ3Er9yMUC14TINk6wNJrvntkcoB0zsvytnUV3bEADUOrsjJilM0QhfvbHK1bG +WVw29JQ8tP6iJyxJkFp1z+neahze1H+s12CscrJt51p2u79/0bClr6/SyWVmURxU +9Jh10DNbAgMBAAECggEBALv7Q+Rf+C7wrQDZF6LUc9CrGfq4CGVy2IGJKgqT/jOF +DO9nI1rv4hNr55sbQNneWtcZaYvht2mrzNlj57zepDjDM7DcFuLBHIuWgLXT/NmC +FyZOo3vXYBlNr8EgT2XfnXAp9UWJCmc2CtUzsIYC4dsmXMeTd8kyc5tUl4r5ybTf +1g+RTck/IGgqdfzpuTsNl79FW2rP9z111Py6dbqgQzhuSAune9dnLFvZst8dyL8j +FStETMxBM6jrCF1UcKXzG7trDHiCdzJ8WUhx6opN/8OasQGndwpXto6FZuBy/AVP +4kVQNpUXImYcLEpva0MqGRHg+YN+c84C71CMchnF4aECgYEA7J2go4CkCcZNKCy5 +R5XVCqNFYRHjekR+UwH8cnCa7pMKKfP+lTCiBrO2q8zwWwknRMyuycS5g/xbSpg1 +L6hi92CV1YQy1/JhlQedekjejNTTuLOPKf78AFNSfc7axDnes2v4Bvcdp9gsbUIO +10cXh0tOSLE7P9y+yC86KQkFAPECgYEA2zO0M2nvbPHv2jjtymY3pflYm0HzhM/T +kPtue3GxOgbEPsHffBGssShBTE3yCOX3aAONXJucMrSAPL9iwUfgfGx6ADdkwBsA +OjDlkxvTbP/9trE6/lsSPtGpWRdJNHqXN4Hx7gXJizRwG7Ym+oHvIIh53aIjdFoE +HLQLpxObuQsCgYAuMQ99G83qQpYpc6GwAeYXL4yJyK453kky9z5LMQRt8rKXQhS/ +F0FqQYc1vsplW0IZQkQVC5yT0Z4Yz+ICLcM0O9zEVAyA78ZxC42Io9UedSXn9tXK +Awc7IQkHmmxGxm1dZYSEB5X4gFEb+zted3h2ZxMfScohS3zLI70c6a/aYQKBgQCU +phRuxUkrTUpFZ1PCbN0R/ezbpLbaewFTEV7T8b6oxgvxLxI6FdZRcSYO89DNvf2w +GLCVe6VKMWPBTlxPDEostndpjCcTq3vU+nHE+BrBkTvh14BVGzddSFsaYpMvNm8z +ojiJHH2XnCDmefkm6lRacJKL/Tcj4SNmv6YjUEXLDwKBgF8WV9lzez3d/X5dphLy +2S7osRegH99iFanw0v5VK2HqDcYO9A7AD31D9nwX46QVYfgEwa6cHtVCZbpLeJpw +qXnYXe/hUU3yn5ipdNJ0Dm/ZhJPDD8TeqhnRRhxbZmsXs8EzfwB2tcUbASvjb3qA +vAaPlOSU1wXqhAsG9aVs8gtL +-----END PRIVATE KEY----- +""", # 3 +"""-----BEGIN CERTIFICATE----- +MIICojCCAYoCAQEwDQYJKoZIhvcNAQELBQAwFzEVMBMGA1UEAwwMbmV3cGJfdGhp +bmd5MB4XDTIwMDEwMjAxNDAzNFoXDTIxMDEwMTAxNDAzNFowFzEVMBMGA1UEAwwM +bmV3cGJfdGhpbmd5MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAzUqQ +M08E7F2ZE99bFHvpsR6LmgIJOOoGMXacTcEUhRF63E6+730FjxER2a30synv9GGS +3G9FstUmfhyimufkbTumri8Novw5CWZQLiE1rmMBI5nPcR2wAzy9z2odR6bfAwms +yyc3IPYg1BEDBPZl0LCQrQRRU/rVOrbCf7IMq+ATazmBg01gXMzq2M953ieorkQX +MsHVR/kyW0Q0yzhYF1OtIqbXxrdiZ+laTLWNqivj/FdegiWPCf8OcqpcpbgEjlDW +gBcC/vre+0E+16nfUV8xHL5jseJMJqfT508OtHxAzp+2D7b54NvYNIvbOAP+F9gj +aXy5mOvjXclK+hNmDwIDAQABMA0GCSqGSIb3DQEBCwUAA4IBAQAjZzTFKG7uoXxm +BPHfQvsKHIB/Cx9zMKj6pLwJzCPHQBzKOMoUen09oq+fb77RM7WvdX0pvFgEXaJW +q/ImooRMo+paf8GOZAuPwdafb2/OGdHZGZ2Cbo/ICGo1wGDCdMvbxTxrDNq1Yae+ +m+2epN2pXAO1rlc7ktRkojM/qi3zXtbLjTs3IoPDXWhYPHdI1ThkneRmvxpzB1rW +2SBqj2snvyI+/3k3RHmldcdOrTlgWQ9hq05jWR8IVtRUFFVn9A+yQC3gnnLIUhwP +HJWwTIPuYW25TuxFxYZXIbnAiluZL0UIjd3IAwxaafvB6uhI7v0K789DKj2vRUkY +E8ptxZH4 +-----END CERTIFICATE----- +-----BEGIN PRIVATE KEY----- +MIIEwAIBADANBgkqhkiG9w0BAQEFAASCBKowggSmAgEAAoIBAQDNSpAzTwTsXZkT +31sUe+mxHouaAgk46gYxdpxNwRSFEXrcTr7vfQWPERHZrfSzKe/0YZLcb0Wy1SZ+ +HKKa5+RtO6auLw2i/DkJZlAuITWuYwEjmc9xHbADPL3Pah1Hpt8DCazLJzcg9iDU +EQME9mXQsJCtBFFT+tU6tsJ/sgyr4BNrOYGDTWBczOrYz3neJ6iuRBcywdVH+TJb +RDTLOFgXU60iptfGt2Jn6VpMtY2qK+P8V16CJY8J/w5yqlyluASOUNaAFwL++t77 +QT7Xqd9RXzEcvmOx4kwmp9PnTw60fEDOn7YPtvng29g0i9s4A/4X2CNpfLmY6+Nd +yUr6E2YPAgMBAAECggEBAIiL6uQl0AmDrBj6vHMghGzp+0MBza6MgngOA6L4JTTp +ToYQ3pEe4D6rxOq7+QHeiBtNd0ilvn9XpVXGqCVOzrIVNiWvaGubRjjJU9WLA1Ct +y4kpekAr1fIhScMXOsh45ub3XXZ27AVBkM5dTlvTpB8uAd0C/TFVqtR10WLsQ99h +Zm9Jczgs/6InYTssnAaqdeCLAf1LbmO4zwFsJfJOeSGGT6WBwlpHwMAgPhg8OLEu +kVWG7BEJ0hxcODk/es/vce9SN7BSyIzNY+qHcGtsrx/o0eO2Av/Z7ltV4Sz6UN1K +0y0OTiDyT/l62U2OugSN3wQ4xPTwlrWl7ZUHJmvpEaECgYEA+w2JoB2i1OV2JTPl +Y0TKSKcZYdwn7Nwh4fxMAJNJ8UbpPqrZEo37nxqlWNJrY/jKX3wHVk4ESSTaxXgF +UY7yKT0gRuD9+vE0gCbUmJQJTwbceNJUu4XrJ6SBtf72WgmphL+MtyKdwV8XltVl +Yp0hkswGmxl+5+Js6Crh7WznPl8CgYEA0VYtKs2YaSmT1zraY6Fv3AIQZq012vdA +7nVxmQ6jKDdc401OWARmiv0PrZaVNiEJ1YV8KxaPrKTfwhWqxNegmEBgA1FZ66NN +SAm8P9OCbt8alEaVkcATveXTeOCvfpZUO3sqZdDOiYLiLCsokHblkcenK85n0yT6 +CzhTbvzDllECgYEAu9mfVy2Vv5OK2b+BLsw0SDSwa2cegL8eo0fzXqLXOzCCKqAQ +GTAgTSbU/idEr+NjGhtmKg/qaQioogVyhVpenLjeQ+rqYDDHxfRIM3rhlD5gDg/j +0wUbtegEHrgOgcSlEW16zzWZsS2EKxq16BoHGx6K+tcS/FOShg5ASzWnuiUCgYEA +sMz+0tLX8aG7CqHbRyBW8FMR9RY/kRMY1Q1+Bw40wMeZfSSSkYYN8T9wWWT/2rqm +qp7V0zJ34BFUJoDUPPH84fok3Uh9EKZYpAoM4z9JP0jREwBWXMYEJnOQWtwxfFGN +DLumgF2Nwtg3G6TL2s+AbtJYH4hxagQl5woIdYmnyzECgYEAsLASpou16A3uXG5J ++5ZgF2appS9Yfrqfh6TKywMsGG/JuiH3djdYhbJFIRGeHIIDb4XEXOHrg/SFflas +If0IjFRh9WCvQxnoRha3/pKRSc3OEka1MR/ZREK/d/LQEPmsRJVzY6ABKqmPAMDD +5CnG6Hz/rP87BiEKd1+3PGp8GCw= +-----END PRIVATE KEY----- +""", # 4 +"""-----BEGIN CERTIFICATE----- +MIICojCCAYoCAQEwDQYJKoZIhvcNAQELBQAwFzEVMBMGA1UEAwwMbmV3cGJfdGhp +bmd5MB4XDTIwMDEwMjAxNDAzNFoXDTIxMDEwMTAxNDAzNFowFzEVMBMGA1UEAwwM +bmV3cGJfdGhpbmd5MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA0sap +75YbbkEL85LFava3FrO1jpgVteQ4NGxxy1Nu9w2hPfMMeCPWjB8UfAwFk+LVPyvW +LAXd1zWL5rGpQ2ytIVQlTraR5EnALA1sMcQYbFz1ISPTYB031bEN/Ch8JWYwCG5A +X2H4D6BC7NgT6YyWDt8vxQnqAisPHQ/OK4ABD15CwkTyPimek2/ufYN2dapg1xhG +IUD96gqetJv9bu0r869s688kADIComsYG+8KKfFN67S3rSHMIpZPuGTtoHGnVO89 +XBm0vNe0UxQkJEGJzZPn0tdec0LTC4GNtTaz5JuCjx/VsJBqrnTnHHjx0wFz8pff +afCimRwA+LCopxPE1QIDAQABMA0GCSqGSIb3DQEBCwUAA4IBAQBOkAnpBb3nY+dG +mKCjiLqSsuEPqpNiBYR+ue/8aVDnOKLKqAyQuyRZttQ7bPpKHaw7pwyCZH8iHnt6 +pMCLCftNSlV2Fa8msRmuf5AiGjUvR1M8VtHWNYE8pedWrJqUgBhF/405B99yd8CT +kQJXKF18LObj7YKNsWRoMkVgqlQzWDMEqbfmy9MhuLx2EZPsTB1L0BHNGGDVBd9o +cpPLUixcc12u+RPMKq8x3KgwsnUf5vX/pCnoGcCy4JahWdDgcZlf0hUKGT7PUem5 +CWW8SMeqSWQX9XpE5Qlm1+W/QXdDXLbbHqDtvBeUy3iFQe3C9RSkp0qdutxkAlFk +f5QHXfJ7 +-----END CERTIFICATE----- +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDSxqnvlhtuQQvz +ksVq9rcWs7WOmBW15Dg0bHHLU273DaE98wx4I9aMHxR8DAWT4tU/K9YsBd3XNYvm +salDbK0hVCVOtpHkScAsDWwxxBhsXPUhI9NgHTfVsQ38KHwlZjAIbkBfYfgPoELs +2BPpjJYO3y/FCeoCKw8dD84rgAEPXkLCRPI+KZ6Tb+59g3Z1qmDXGEYhQP3qCp60 +m/1u7Svzr2zrzyQAMgKiaxgb7wop8U3rtLetIcwilk+4ZO2gcadU7z1cGbS817RT +FCQkQYnNk+fS115zQtMLgY21NrPkm4KPH9WwkGqudOccePHTAXPyl99p8KKZHAD4 +sKinE8TVAgMBAAECggEALU5EotoqJUXYEtAenUJQ0pFoWjE4oXNf3Wzd/O1/MZ19 +ZjqDGKPjbxUTKyLOZB5i5gQ/MhFEwQiifMD9eB+5CyvyJPw7Wc28f/uWoQ/cjBZj +Hm979PHy2X0IW4Y8QTG462b/cUE2t+0j1ZMQnKf6bVHuC7V41mR5CC8oitMl5y5g +34yJmWXlIA0ep/WotLMqvil6DnSM/2V8Ch4SxjnzPpjbe4Kj+woucGNr4UKstZER +8iuHTsR64LjoGktRnnMwZxGZQI7EC428zsliInuWMdXe//w2chLdkirqpSrIQwSZ +3jNWStqBXGYaRg5Z1ilBvHtXxkzDzbAlzRBzqfEwwQKBgQDqYdMRrzHJaXWLdsyU +6jAuNX9tLh7PcicjP93SbPujS6mWcNb+D/au+VhWD+dZQDPRZttXck7wvKY1lw1V +MK0TYI7ydf8h3DFx3Mi6ZD4JVSU1MH233C3fv/FHenDoOvMXXRjUZxaRmuzFJvzt +6QlKIfSvwT+1wrOACNfteXfZUQKBgQDmN3Uuk01qvsETPwtWBp5RNcYhS/zGEQ7o +Q4K+teU453r1v8BGsQrCqulIZ3clMkDru2UroeKn1pzyVAS2AgajgXzfXh3VeZh1 +vHTLP91BBYZTTWggalEN4aAkf9bxX/hA+9Bw/dzZcQW2aNV7WrYuCSvp3SDCMina +anQq/PaSRQKBgHjw23HfnegZI89AENaydQQTFNqolrtiYvGcbgC7vakITMzVEwrr +/9VP0pYuBKmYKGTgF0RrNnKgVX+HnxibUmOSSpCv9GNrdJQVYfpT6XL1XYqxp91s +nrs7FuxUMNiUOoWOw1Yuj4W4lH4y3QaCXgnDtbfPFunaOrdRWOIv8HjRAoGAV3NT +mSitbNIfR69YIAqNky3JIJbb42VRc1tJzCYOd+o+pCF96ZyRCNehnDZpZQDM9n8N +9GAfWEBHCCpwS69DVFL422TGEnSJPJglCZwt8OgnWXd7CW05cvt1OMgzHyekhxLg +4Dse7J5pXBxAlAYmVCB5xPGR4xLpISX1EOtcwr0CgYEA5rA2IUfjZYb4mvFHMKyM +xWZuV9mnl3kg0ULttPeOl3ppwjgRbWpyNgOXl8nVMYzxwT/A+xCPA18P0EcgNAWc +frJqQYg3NMf+f0K1wSaswUSLEVrQOj25OZJNpb21JEiNfEd5DinVVj4BtVc6KSpS +kvjbn2WhEUatc3lPL3V0Fkw= +-----END PRIVATE KEY----- +""", # 5 +"""-----BEGIN CERTIFICATE----- +MIICojCCAYoCAQEwDQYJKoZIhvcNAQELBQAwFzEVMBMGA1UEAwwMbmV3cGJfdGhp +bmd5MB4XDTIwMDEwMjAxNTExM1oXDTIxMDEwMTAxNTExM1owFzEVMBMGA1UEAwwM +bmV3cGJfdGhpbmd5MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA1c5y +S9IZHF9MIuwdafzhMkgP37I3RVpHEbpnPwnLFqSWelS5m2eDkwWd5SkfGjrmQ5q0 +PEpqLlh3zHGw9yQjnHS3CCS1PwQ1kmwvpIK3HM5y8GM7ry1zkam8ZR4iX6Y7VG9g +9mhiVVFoVhe1gHeiC/3Mp6XeNuEiD0buM+8qZx9B21I+iwzy4wva7Gw0fJeq9G1c +lq2rhpD1LlIEodimWOi7lOEkNmUiO1SvpdrGdxUDpTgbdg6r5pCGjOXLd74tAQHP +P/LuqRNJDXtwvHtLIVQnW6wjjy4oiWZ8DXOdc9SkepwQLIF5Wh8O7MzF5hrd6Cvw +SOD3EEsJbyycAob6RwIDAQABMA0GCSqGSIb3DQEBCwUAA4IBAQBDNcbKVUyGOAVm +k3iVuzkkkymlTAMm/gsIs6loLJrkSqNg160FdVKJoZFjQtqoqLgLrntdCJ377nZ9 +1i+yzbZsA4DA7nxj0IEdnd7rRYgGLspGqWeKSTROATeT4faLTXenecm0v2Rpxqc7 +dSyeZJXOd2OoUu+Q64hzXCDXC6LNM+xZufxV9qv+8d+CipV6idSQZaUWSVuqFCwD +PT0R4eWfkMMaM8QqtNot/hVCEaKT+9rG0mbpRe/b/qBy5SR0u+XgGEEIV+33L59T +FXY+DpI1Dpt/bJFoUrfj6XohxdTdqYVCn1F8in98TsRcFHyH1xlkS3Y0RIiznc1C +BwAoGZ4B +-----END CERTIFICATE----- +-----BEGIN PRIVATE KEY----- +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDVznJL0hkcX0wi +7B1p/OEySA/fsjdFWkcRumc/CcsWpJZ6VLmbZ4OTBZ3lKR8aOuZDmrQ8SmouWHfM +cbD3JCOcdLcIJLU/BDWSbC+kgrccznLwYzuvLXORqbxlHiJfpjtUb2D2aGJVUWhW +F7WAd6IL/cynpd424SIPRu4z7ypnH0HbUj6LDPLjC9rsbDR8l6r0bVyWrauGkPUu +UgSh2KZY6LuU4SQ2ZSI7VK+l2sZ3FQOlOBt2DqvmkIaM5ct3vi0BAc8/8u6pE0kN +e3C8e0shVCdbrCOPLiiJZnwNc51z1KR6nBAsgXlaHw7szMXmGt3oK/BI4PcQSwlv +LJwChvpHAgMBAAECggEBAK0KLeUBgIM++Y7WDCRInzYjrn08bpE5tIU7mO4jDfQg +dw1A3wtQZuOpyxW6B0siWlRis/aLv44M2cBkT3ZmEFBDAhOcKfh7fqQn3RNHG847 +pDi8B4UKwxskBa7NCcLh9eirUA19hABLJ6dt/t6fdE5CNc2FZ+iAoyE8JfNwYKAd +6Fa3HqUBPNWt8ryj4ftgpMNBdfmLugEM4N20SXJA28hOq2lUcwNKQQ1xQrovl0ig +iMbMWytV4gUPKC9Wra66OYIkk/K8teiUNIYA4JwAUVTs1NEWoyfwUTz1onutCkMl +5vY7JAqRoDWoSUX6FI+IHUdyqPAMdOMhC37gjrxoo2ECgYEA7trDMu6xsOwEckDh +iz148kejMlnTTuCNetOFBw3njFgxISx0PrDLWmJmnHMxPv9AAjXYb2+UCCm3fj6Q +OB8o4ZJm0n504qbFHcb2aI22U5hZ99ERvqx8WBnJ2RarIBmg06y0ktxq8gFR2qxF +0hWAOcDn1DWQ8QI0XBiFFcJTGtcCgYEA5SdlIXRnVZDKi5YufMAORG9i74dXUi0Y +02UoVxJ+q8VFu+TT8wrC5UQehG3gX+79Cz7hthhDqOSCv6zTyE4Evb6vf9OLgnVe +E5iLF033zCxLSS9MgiZ+jTO+wK3RsapXDtGcSEk2P82Pj5seNf4Ei1GNCRlm1DbX +71wlikprHhECgYABqmLcExAIJM0vIsav2uDiB5/atQelMCmsZpcx4mXv85l8GrxA +x6jTW4ZNpvv77Xm7yjZVKJkGqYvPBI6q5YS6dfPjmeAkyHbtazrCpeJUmOZftQSD +qN5BGwTuT5sn4SXe9ABaWdEhGONCPBtMiLvZK0AymaEGHTbSQZWD/lPoBwKBgGhk +qg2zmd/BNoSgxkzOsbE7jTbR0VX+dXDYhKgmJM7b8AjJFkWCgYcwoTZzV+RcW6rj +2q+6HhizAV2QvmpiIIbQd+Mj3EpybYk/1R2ox1qcUy/j/FbOcpihGiVtCjqF/2Mg +2rGTqMMoQl6JrBmsvyU44adjixTiZz0EHZYCkQoBAoGBAMRdmoR4mgIIWFPgSNDM +ISLJxKvSFPYDLyAepLfo38NzKfPB/XuZrcOoMEWRBnLl6dNN0msuzXnPRcn1gc1t +TG7db+hivAyUoRkIW3dB8pRj9dDUqO9OohjKsJxJaQCyH5vPkQFSLbTIgWrHhU+3 +oSPiK/YngDV1AOmPDH7i62po +-----END PRIVATE KEY----- +""", #6 +"""-----BEGIN CERTIFICATE----- +MIICojCCAYoCAQEwDQYJKoZIhvcNAQELBQAwFzEVMBMGA1UEAwwMbmV3cGJfdGhp +bmd5MB4XDTIwMDEwMjAxNTExMloXDTIxMDEwMTAxNTExMlowFzEVMBMGA1UEAwwM +bmV3cGJfdGhpbmd5MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAojGu +fQaTVT9DJWJ/zogGfrryEJXYVy9c441O5MrLlRx7nCIWIUs2NEhHDJdqJjYOTdmk +K98VhdMpDPZwxjgvvZrh43lStBRIW3zZxv747rSl2VtpSqD/6UNWJe5u4SR7oga4 +JfITOKHg/+ASxnOxp/iu6oT6jBL6T7KSPh6Rf2+it2rsjhktRreFDJ2hyroNq1w4 +ZVNCcNPgUIyos8u9RQKAWRNchFh0p0FCS9xNrn3e+yHnt+p6mOOF2gMzfXT/M2hq +KQNmc5D3yNoH2smWoz7F3XsRjIB1Ie4VWoRRaGEy7RwcwiDfcaemD0rQug6iqH7N +oomF6f3R4DyvVVLUkQIDAQABMA0GCSqGSIb3DQEBCwUAA4IBAQB/8SX6qyKsOyex +v3wubgN3FPyU9PqMfEzrFM6X5sax0VMVbSnekZrrXpdnXYV+3FBu2GLLQc900ojj +vKD+409JIriTcwdFGdLrQPTCRWkEOae8TlXpTxuNqJfCPVNxFN0znoat1bSRsX1U +K0mfEETQ3ARwlTkrF9CM+jkU3k/pnc9MoCLif8P7OAF38AmIbuTUG6Gpzy8RytJn +m5AiA3sds5R0rpGUu8mFeBpT6jIA1QF2g+QNHKOQcfJdCdfqTjKw5y34hjFqbWG9 +RxWGeGNZkhC/jADCt+m+R6+hlyboLuIcVp8NJw6CGbr1+k136z/Dj+Fdhm6FzF7B +qULeRQJ+ +-----END CERTIFICATE----- +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCiMa59BpNVP0Ml +Yn/OiAZ+uvIQldhXL1zjjU7kysuVHHucIhYhSzY0SEcMl2omNg5N2aQr3xWF0ykM +9nDGOC+9muHjeVK0FEhbfNnG/vjutKXZW2lKoP/pQ1Yl7m7hJHuiBrgl8hM4oeD/ +4BLGc7Gn+K7qhPqMEvpPspI+HpF/b6K3auyOGS1Gt4UMnaHKug2rXDhlU0Jw0+BQ +jKizy71FAoBZE1yEWHSnQUJL3E2ufd77Iee36nqY44XaAzN9dP8zaGopA2ZzkPfI +2gfayZajPsXdexGMgHUh7hVahFFoYTLtHBzCIN9xp6YPStC6DqKofs2iiYXp/dHg +PK9VUtSRAgMBAAECggEANjn0A3rqUUr4UQxwfIV/3mj0O1VN4kBEhxOcd+PRUsYW +EapXycPSmII9ttj8tU/HUoHcYIqSMI7bn6jZJXxtga/BrALJAsnxMx031k8yvOQK +uvPT7Q6M4NkReVcRHRbMeuxSLuWTRZDhn8qznEPb9rOvD1tsRN6nb3PdbwVbUcZh +2F6JDrTyI/Df6nrYQAWOEe2ay7tzgrNYE4vh+DW7oVmyHRgFYA+DIG5Q+7OVWeW5 +bwYYPKlo4/B0L+GfMKfMVZ+5TvFWAK0YD1e/CW1Gv+i/8dWm4O7UNGg5mTnrIcy1 +g5wkKbyea02/np2B/XBsSWXDl6rTDHL7ay0rH2hjEQKBgQDMKSm3miQTIcL/F2kG +ieapmRtSc7cedP967IwUfjz4+pxPa4LiU47OCGp1bmUTuJAItyQyu/5O3uLpAriD +PTU+oVlhqt+lI6+SJ4SIYw01/iWI3EF2STwXVnohWG1EgzuFM/EqoB+mrodNONfG +UmP58vI9Is8fdugXgpTz4Yq9pQKBgQDLYJoyMVrYTvUn5oWft8ptsWZn6JZXt5Bd +aXh+YhNmtCrSORL3XjcH4yjlcn7X8Op33WQTbPo7QAJ1CumJzAI88BZ/8za638xb +nLueviZApCt0bNMEEdxDffxHFc5TyHE+obMKFfApbCnD0ggO6lrZ8jK9prArLOCp +mRU9SSRffQKBgAjoBszeqZI4F9SfBdLmMyzU5A89wxBOFFMdfKLsOua1sBn627PZ +51Hvpg1HaptoosfujWK1NsvkB0wY9UmsYuU/jrGnDaibnO4oUSzN/WaMlsCYszZg +zYFLIXrQ67tgajlOYcf1Qkw4MujYgPlC4N+njI/EM/rwagGUjcDx5uaNAoGASyqz +EuYG63eTSGH89SEaohw0+yaNmnHv23aF4EAjZ4wjX3tUtTSPJk0g6ly84Nbb8d1T +hZJ7kbaAsf2Mfy91jEw4JKYhjkP05c8x0OP6g12p6efmvdRUEmXX/fXjQjgNEtb0 +sz+UedrOPN+9trWLSo4njsyyw+JcTpKTtQj5dokCgYEAg9Y3msg+GvR5t/rPVlKd +keZkrAp0xBJZgqG7CTPXWS1FjwbAPo7x4ZOwtsgjCuE52lar4j+r2Il+CDYeLfxN +h/Jfn6S9ThUh+B1PMvKMMnJUahg8cVL8uQuBcbAy8HPRK78WO2BTnje44wFAJwTc +0liuYqVxZIRlFLRl8nGqog8= +-----END PRIVATE KEY----- +""", #7 +"""-----BEGIN CERTIFICATE----- +MIICojCCAYoCAQEwDQYJKoZIhvcNAQELBQAwFzEVMBMGA1UEAwwMbmV3cGJfdGhp +bmd5MB4XDTIwMDEwMjAxNTExMloXDTIxMDEwMTAxNTExMlowFzEVMBMGA1UEAwwM +bmV3cGJfdGhpbmd5MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAu9oO +cFlNukUcLfFrfkEaUiilcHLmn5OokQbj95CGd2ehQCCVwrkunYLBisthRaancFFb +/yM998B0IUsKTsoLi5DAN3/SkSm6GiQIGO05E4eBPljwJ61QQMxh8+1TwQ9HTun1 +ZE1lhVN1aRmI9VsbyTQLjXh9OFNLSJEKb29mXsgzYwYwNOvo+idzXpy4bMyNoGxY +Y+s2FIKehNHHCv4ravDn8rf6DtDOvyN4d0/QyNws9FpAZMXmLwtBJ9exOqKFW43w +97NxgdNiTFyttrTKTi0b+9v3GVdcEZw5b2RMIKi6ZzPof6/0OlThK6C3xzFK3Bp4 +PMjTfXw5yyRGVBnZZwIDAQABMA0GCSqGSIb3DQEBCwUAA4IBAQA4Ms6LqzMu757z +bxISiErRls6fcnq0fpSmiPNHNKM7YwG9KHYwPT6A0UMt30zDwNOXCQBI19caGeeO +MLPWa7Gcqm2XZB2jQwvLRPeFSy9fm6RzJFeyhrh/uFEwUetwYmi/cqeIFDRDBQKn +bOaXkBk0AaSmI5nRYfuqpMMjaKOFIFcoADw4l9wWhv6DmnrqANzIdsvoSXi5m8RL +FcZQDZyHFlHh3P3tLkmQ7ErM2/JDwWWPEEJMlDm/q47FTOQSXZksTI3WRqbbKVv3 +iQlJjpgi9yAuxZwoM3M4975iWH4LCZVMCSqmKCBt1h9wv4LxqX/3kfZhRdy1gG+j +41NOSwJ/ +-----END CERTIFICATE----- +-----BEGIN PRIVATE KEY----- +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQC72g5wWU26RRwt +8Wt+QRpSKKVwcuafk6iRBuP3kIZ3Z6FAIJXCuS6dgsGKy2FFpqdwUVv/Iz33wHQh +SwpOyguLkMA3f9KRKboaJAgY7TkTh4E+WPAnrVBAzGHz7VPBD0dO6fVkTWWFU3Vp +GYj1WxvJNAuNeH04U0tIkQpvb2ZeyDNjBjA06+j6J3NenLhszI2gbFhj6zYUgp6E +0ccK/itq8Ofyt/oO0M6/I3h3T9DI3Cz0WkBkxeYvC0En17E6ooVbjfD3s3GB02JM +XK22tMpOLRv72/cZV1wRnDlvZEwgqLpnM+h/r/Q6VOEroLfHMUrcGng8yNN9fDnL +JEZUGdlnAgMBAAECggEALlZdlW0R9U6y4spYf65Dddy84n4VUWu0+wE+HoUyBiYz +6oOfLYdMbmIgp8H/XpT7XINVNBxXXtPEUaoXAtRoAKdWItqO8Gvgki4tKSjrGVwl +j2GU69SepT1FNExoiojgSCEB/RnyXu71WVWJKSyuL/V8nAsKqGgze9T7Q/2wvNQt +SQqLxZlrWF0P8WqaAiSrHV4GnDrdeF+k1KBo2+pSaDNv6cNwOyVG8EII9tqhF8kj +6nD6846ish6OqmlSisaSGopJZL1DCQzszFMxKd2+iBDY7Kn6hVIhRaNnaZUFhpKM +dNh6hBqOycMepAp0sz5pdo+fxpifkoR/cPWgyC3QkQKBgQDixe9VsiZ7u2joxF/9 +JcAExKhqE28OUmIwt6/j+uzYShxN6Oo9FUo3ICtAPCCFsjhvb3Qum7FspmxrqtNy +fzclibZJPO8ey2PzqaiOfiVfgJmNSvoCOdgM4OqFLtRO6eSTzhJeI4VPrPcq/5la +0FuOi1WZs/Au9llqLqGSDH3UAwKBgQDUD/bSJbOk5SvNjFtFm0ClVJr66mJ5e4uN +4VGv8KGFAJ+ExIxujAukfKdwLjS1wEy2RePcshfT8Y9FVh/Q1KzzrQi3Gwmfq1G6 +Dpu2HlJpaZl+9T81x2KS8GP3QNczWMe2nh7Lj+6st+b4F+6FYbVTFnHaae27sXrD +XPX15+uxzQKBgGy+pBWBF4kwBo/QU4NuTdU7hNNRPGkuwl1ASH1Xv6m8aDRII8Nk +6TDkITltW98g5oUxehI7oOpMKCO9SCZYsNY0YpBeQwCOYgDfc6/Y+A0C+x9RO/BD +UsJiPLPfD/pDmNPz9sTj3bKma+RXq29sCOujD0pkiiHLCnerotkJWnGHAoGAAkCJ +JoIv/jhQ1sX+0iZr8VWMr819bjzZppAWBgBQNtFi4E4WD7Z9CSopvQ9AkA2SwvzL +BrT9e8q88sePXvBjRdM4nHk1CPUQ0SEGllCMH4J3ltmT6kZLzbOv3BhcMLdop4/W +U+MbbcomMcxPRCtdeZxraR5m3+9qlliOZCYqYqECgYA5eLdxgyHxCS33QGFHRvXI +TLAHIrr7wK1xwgkmZlLzYSQ8Oqh1UEbgoMt4ulRczP2g7TCfvANw2Sw0H2Q5a6Fj +cnwVcXJ38DLg0GCPMwzE8dK7d8tKtV6kGiKy+KFvoKChPjE6uxhKKmCJaSwtQEPS +vsjX3iiIgUQPsSz8RrNFfQ== +-----END PRIVATE KEY----- +""", #8 +"""-----BEGIN CERTIFICATE----- +MIICojCCAYoCAQEwDQYJKoZIhvcNAQELBQAwFzEVMBMGA1UEAwwMbmV3cGJfdGhp +bmd5MB4XDTIwMDEwMjAxNTExMloXDTIxMDEwMTAxNTExMlowFzEVMBMGA1UEAwwM +bmV3cGJfdGhpbmd5MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA5DNu +CKhhl6wCbgoCkFemwJh3ATbAjhInHpvQWIFDfSK1USElCKxqosIxiBQCx3Zs2d/U +GeIA7QAM2atNdXaateacEaKMmGE9LEtO0Dg5lmT43WzmGkG9NmCwK3JjAekc5S9d +HKNtEQo7o8RKfj81zlDSq2kzliy98cimk24VBBGkS2Cn7Vy/mxMCqWjQazTXbpoS +lXw6LiY5wFXQmXOB5GTSHvqyCtBQbOSSbJB77z/fm7bufTDObufTbJIq53WPt00Y +f+JNnzkX1X0MaBCUztoZwoMaExWucMe/7xsQ46hDn6KB4b0lZk+gsK45QHxvPE1R +72+ZkkIrGS/ljIKahQIDAQABMA0GCSqGSIb3DQEBCwUAA4IBAQDib1653CneSmy2 +gYzGeMlrI05Jqo3JuHNMQHzAjIrb4ee57VA4PTQa1ygGol/hVv6eTvZr3p2ospDS +5Kfwj1HLO4jSMX1Bnm1FG0naQogz2CD3xfYjbYOVRhAxpld1MNyRveIOhDRARY7N +XNAaNPZ1ALrwbENSYArr18xDzgGWe/dgyRCEpCFIsztiA+7jGvrmAZgceIE8K3h3 +fkvNmXBH58ZHAGTiyRriBZqS+DXrBrQOztXSJwFnOZnRt6/efeBupt8j5hxVpBLW +vtjpBc23uUcbbHOY2AW2Bf+vIr4/LmJ/MheKV+maa2990vmC93tvWlFfc74mgUkW +HJfXDmR6 +-----END CERTIFICATE----- +-----BEGIN PRIVATE KEY----- +MIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQDkM24IqGGXrAJu +CgKQV6bAmHcBNsCOEicem9BYgUN9IrVRISUIrGqiwjGIFALHdmzZ39QZ4gDtAAzZ +q011dpq15pwRooyYYT0sS07QODmWZPjdbOYaQb02YLArcmMB6RzlL10co20RCjuj +xEp+PzXOUNKraTOWLL3xyKaTbhUEEaRLYKftXL+bEwKpaNBrNNdumhKVfDouJjnA +VdCZc4HkZNIe+rIK0FBs5JJskHvvP9+btu59MM5u59NskirndY+3TRh/4k2fORfV +fQxoEJTO2hnCgxoTFa5wx7/vGxDjqEOfooHhvSVmT6CwrjlAfG88TVHvb5mSQisZ +L+WMgpqFAgMBAAECggEABTdPuo7uvCLIY2+DI319aEWT4sk3mYe8sSxqlLtPqZqT +fmk9iXc3cMTzkOK0NY71af19waGy17f6kzchLCAr5SCCTLzkbc87MLn/8S530oI4 +VgdZMxxxkL6hCD0zGiYT7QEqJa9unMcZGeMwuLYFKtQaHKTo8vPO26n0dMY9YLxj +cNUxsKLcKk8dbfKKt4B4fZgB7nU0BG9YbKYZ3iZ7/3mG+6jA6u+VYc/WHYQjTmpL +oLFN7NOe3R7jIx/kJ1OqNWqsFoLpyiiWd1Mr0l3EdD1kCudptMgD8hd++nx2Yk2w +K4+CpOVIN/eCxDDaAOJgYjCtOayVwUkDAxRRt9VnAQKBgQD5s1j6RJtBNTlChVxS +W3WpcG4q8933AiUY/Chx0YTyopOiTi7AGUaA8AOLFBcO2npa+vzC+lvuOyrgOtVW +sD10H2v5jNKlbeBp+Q9rux2LAyp4TvzdXWKhVyZrdtITF0hn6vEYNp7MtyWRFb1O +3Ie5HQBPHtzllFOMynacjOdjpQKBgQDp9TrbfOmwGWmwPKmaFKuy8BKxjJM+ct0X +4Xs1uSy9Z9Y8QlDNbNaooI8DA1NY0jDVHwemiGC4bYsBNKNRcbI0s2nr0hQMft42 +P/NpugHv0YXiVz+5bfim4woTiHHbfREqchlIGo3ryClAiDU9fYZwTOtb9jPIhX3G +9v+OsoMlYQKBgQDJUQW90S5zJlwh+69xXvfAQjswOimNCpeqSzK4gTn0/YqV4v7i +Nf6X2eqhaPMmMJNRYuYCtSMFMYLiAc0a9UC2rNa6/gSfB7VU+06phtTMzSKimNxa +BP6OIduB7Ox2I+Fmlw8GfJMPbeHF1YcpW7e5UV58a9+g4TNzYZC7qwarWQKBgQCA +FFaCbmHonCD18F/REFvm+/Lf7Ft3pp5PQouXH6bUkhIArzVZIKpramqgdaOdToSZ +SAGCM8rvbFja8hwurBWpMEdeaIW9SX8RJ/Vz/fateYDYJnemZgPoKQcNJnded5t8 +Jzab+J2VZODgiTDMVvnQZOu8To6OyjXPRM0nK6cMQQKBgQDyX44PHRRhEXDgJFLU +qp2ODL54Qadc/thp2m+JmAvqmCCLwuYlGpRKVkLLuZW9W6RlVqarOC3VD3wX5PRZ +IsyCGLi+Jbrv9JIrYUXE80xNeQVNhrrf02OW0KHbqGxRaNOmp1THPw98VUGR2J/q +YAp6XUXU7LEBUrowye+Ty2o7Lg== +-----END PRIVATE KEY----- +""", #9 +"""-----BEGIN CERTIFICATE----- +MIICojCCAYoCAQEwDQYJKoZIhvcNAQELBQAwFzEVMBMGA1UEAwwMbmV3cGJfdGhp +bmd5MB4XDTIwMDEwMjAxNTExMVoXDTIxMDEwMTAxNTExMVowFzEVMBMGA1UEAwwM +bmV3cGJfdGhpbmd5MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA1k2R +PWYihftppo3CoxeseFwgg7guxZVkP7aAur5uBzSeAB7sBG1G2bRrwMX71S4xPwot +zYiEoxUrTStUqEKjL2aozfHsXnHZ7kwwUgZFDZUg+ve2tZDA3HCUr4tLYKlyFqpx +2nCouc45MjQ4wAxRl4rQxIUG2uSTzvP+xXtjoJYMIEEyCpcsRXfqfVkEUe9nrPsF +0Ibzk7Cyt75HDI4uEzBuHux0DYuGy6R02jz/vf/dIZ4WepjSY06xpblTHZgieDRX +fU2+YOcvb0eDHyA8Q5p8ropK71MNIP5+kffFd90SVr4EkCA8S+cd6FdKQasRr+jF +9MUhMS4ObvlrYTG+hwIDAQABMA0GCSqGSIb3DQEBCwUAA4IBAQCy62MZ3+59/VpX +c9Hsmb4/BMWt0irLJit4w4SkuYKGFLCMKZI4LN4pEkXaiE1eqF2DNS1qOvl5luty +Zz4oggrqilwuFeH98o9Zeg9SYnouuypORVP/3DPbJF/jiQg5J8kJb1sy+BjRiT8I +5X6/cCBYT+MljFz5tpqWOtWTgA30e1BV8JFj8F4dgUcWsAVT/I4l9zgMLUnhcO6E +wLtEE0I6aT1RHJB28ndwJzj4La98Oirw7LAEAWbExWYB90ypLaGY+JVJe3f5fijC +fJpQ2mbs4syXDmb5bU2C2pGPTKZPcyx15iQrq1uHInD0facOw+pmllAFxuG96lA1 ++o2VzKwP +-----END CERTIFICATE----- +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDWTZE9ZiKF+2mm +jcKjF6x4XCCDuC7FlWQ/toC6vm4HNJ4AHuwEbUbZtGvAxfvVLjE/Ci3NiISjFStN +K1SoQqMvZqjN8execdnuTDBSBkUNlSD697a1kMDccJSvi0tgqXIWqnHacKi5zjky +NDjADFGXitDEhQba5JPO8/7Fe2OglgwgQTIKlyxFd+p9WQRR72es+wXQhvOTsLK3 +vkcMji4TMG4e7HQNi4bLpHTaPP+9/90hnhZ6mNJjTrGluVMdmCJ4NFd9Tb5g5y9v +R4MfIDxDmnyuikrvUw0g/n6R98V33RJWvgSQIDxL5x3oV0pBqxGv6MX0xSExLg5u ++WthMb6HAgMBAAECggEAeCyRSNwQeg/NZD/UqP6qkegft52+ZMBssinWsGH/c3z3 +KVwtwCHDfGvnjPe5TAeWSCKeIsbukkFZwfGNjLmppvgrqymCAkhYDICfDDBF4uMA +1pu40sJ01Gkxh+tV/sOmnb1BEVzh0Sgq/NM6C8ActR18CugKOw+5L3G2KeoSqUbT +2hcPUsnik10KwqW737GQW4LtEQEr/iRmQkxI3+HBzvPWjFZzjOcpUph+FW5TXtaU +T26mt1j+FjbdvvhCuRMY/VZBJ5h1RKU95r57F1AjW/C0RRJ8FxR1CeSy4IlmQBrh +6wAa3Tdm0k/n4ZspC9bF5eVTJEtb0AohiYZrIa8MuQKBgQD8yjCLYa41H304odCx +NwPRJcmlIk5YGxPrhHAT9GEgU6n/no7YMVx1L7fNLcMjAyx54jauEU7J19Aki7eV +SIdU9TwqmkOAFfM6TOEJZiOi66gABOxeK2yDyfmR6Apaw3caku4O058t4KVwHSCB +DanYCMzxCBqS9jUTTyAh0fMg6wKBgQDZBkIukg3FKPor5LzkUXIKnNHYPfHbERHw +piWS6GZwqhuWNlOCWxiBR4rEUU/RbFQZw/FCi5OuAk2lBC0LBmC0/Sz4/+xDdCbv +uNhMOTRcy9nFVpmpIWCx4N/KmXHEuFxli/JNXux7iki74AVC9VPrAt/kCvwf06Df +oDb8ljdR1QKBgQChVOD6c5Lc8IXYeN1Z3IShHH6+11AsxstFyjZFZff+y6Z5L1Z2 +/7nESHoDhqs9Uy81cnv3R7CC/Ssnx8uYiLtmK0UE44Mk4d1jXeFZQEiKF+AWcw3v +Y8NTsLmItxC0sH75BMDN0Z2LiA3Nqaku8+trpuI1Cjj7hgqFkkAtlXKXlQKBgBMb +c/Q5s7CqHOyEZQUNDqdUiz0opwSMijHPzvsSLwK4V1lwSwXtE0k+jT8fkZF0oirq +j3E2bLgjR8bBiV2xIA6PQ8hgb+K4dT0h3xlG6A9Le07egwTbBXJjxBBIVjXlrWzb +V2fsdZGi6ShxXsU4aD0GscOYG/6JWV6W8oBmkVRJAoGAepIZ+OYmFjb7uxdh4EtP +hluEtx5bLOLuo6c0S149omUXUhbsuyzTZS6Ip9ySDMnK3954c4Q4WJ4yQKixQNVq +78aDfy4hP/8TE/Q9CRddUof2P33PJMhVNqzTRYMpqV+zxifvtw3hoDTLKHTQxCR2 +M1+O4VvokU5pBqUpGXiMDfs= +-----END PRIVATE KEY----- +""", #10 +"""-----BEGIN CERTIFICATE----- +MIICojCCAYoCAQEwDQYJKoZIhvcNAQELBQAwFzEVMBMGA1UEAwwMbmV3cGJfdGhp +bmd5MB4XDTIwMDEwMjAxNTExMVoXDTIxMDEwMTAxNTExMVowFzEVMBMGA1UEAwwM +bmV3cGJfdGhpbmd5MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAnbCU +M37hG7zrCyyJEI6pZmOomnI+CozbP5KAhWSV5y7R5H6lcAEG2UDV+lCUxHT2ufOa +i1H16bXyBt7VoMTHIH50S58NUCUEXcuRWVR16tr8CzcTHQAkfIrmhY2XffPilX7h +aw35UkoVmXcqSDNNJD6jmvWexvmbhzVWW8Vt5Pivet2/leVuqPXB54/alSbkC74m +x6X5XKQc6eyPsb1xvNBuiSpFzdqbEn7lUwj6jFTkh9tlixgmgx+J0XoQXbawyrAg +rcIQcse/Ww+KBA1KSccFze+XBTbIull4boYhbJqkb6DW5bY7/me2nNxE9DRGwq+S +kBsKq3YKeCf8LEhfqQIDAQABMA0GCSqGSIb3DQEBCwUAA4IBAQAD+tWGFhINYsWT +ibKWlCGgBc5uB7611cLCevx1yAL6SaOECVCQXzaaXIaETSbyY03UO2yBy3Pl10FV +GYXLrAWTFZsNVJm55XIibTNw1UBPNwdIoCSzAYuOgMF0GHhTTQU0hNYWstOnnE2T +6lSAZQZFkaW4ZKs6sUp42Em9Bu99PehyIgnw14qb9NPg5qKdi2GAvkImZCrGpMdK +OF31U7Ob0XQ0lxykcNgG4LlUACd+QxLfNpmLBZUGfikexYa1VqBFm3oAvTt8ybNQ +qr7AKXDFnW75aCBaMpQWzrstA7yYZ3D9XCd5ZNf6d08lGM/oerDAIGnZOZPJgs5U +FaWPHdS9 +-----END CERTIFICATE----- +-----BEGIN PRIVATE KEY----- +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCdsJQzfuEbvOsL +LIkQjqlmY6iacj4KjNs/koCFZJXnLtHkfqVwAQbZQNX6UJTEdPa585qLUfXptfIG +3tWgxMcgfnRLnw1QJQRdy5FZVHXq2vwLNxMdACR8iuaFjZd98+KVfuFrDflSShWZ +dypIM00kPqOa9Z7G+ZuHNVZbxW3k+K963b+V5W6o9cHnj9qVJuQLvibHpflcpBzp +7I+xvXG80G6JKkXN2psSfuVTCPqMVOSH22WLGCaDH4nRehBdtrDKsCCtwhByx79b +D4oEDUpJxwXN75cFNsi6WXhuhiFsmqRvoNbltjv+Z7ac3ET0NEbCr5KQGwqrdgp4 +J/wsSF+pAgMBAAECggEAPSu1ofBTRN5ZU4FYPlsJLdX1Hsy4coFHv/aF8rkdSYwp +EflrFfLgBEEZgLvnqfoxh9sPFYKa4amaFL42ouIS2PEVDgzKLk/dzMDeRof0IkIG +yhb4TCS1ArcjS6WsociNGi8ZJN1L3Xctv9WxSkbUYv4Fm2Qyzr8fbSjssjb5NXwD +K11fsj6Pfy/mQrI0TSTlzWC7ARIlCMTWQ8G8zEU6bMFIG6DMjt2J4VgYVXUKetZA +VPuS+pwkH2obQe6FLRuiNxH4GitVAASYPea6foER4AggBMRp8q8F6+WssjoyEORb +0sJxmnxhznoTRMCuTsUj6XMgmOTOnA3lQXsIB0DYcQKBgQDO6mMRVVFWzgkE9Q5/ +36n06KvGYF9TCRDL9vRC8kCqcGd1Hy6jRj0D8049KUHaN74pfWg6gsQjPkKzwKnC +vxNl72tVvLqm7Fo531BGfKK/46ZvxeWMMraNW4+9LhwMPu2LN5OEdwwCgyaURpxh +ktCp+RrGjz08Kn82X1jJPdwxDQKBgQDDGMvZ7ZUDGq5+RJkmHJ58lQtiaMZclmYV +R9YwOxJV6ino3EYrGOtUkqiemgAACdMWE/JMJlB1/JINawJwUsZ2XDp/9jNLPgLc +gphCmagaO34U/YMaJbJIK2gkCX7p8EcD+x45qWa0bEMPW38QfN/qQdUPjNmpuIiI +Zleyl1TqDQKBgQCvIoat0ighsAzETGN0aqzhJdrW8xVcJA06hpFi5MdFPBTldno0 +KqxUXqj3badWe94SIhqJg8teBUHSAZ3uv2o82nRgQnk99km8OD8rGi1q+9YRP1C2 +5OnNJhW4y4FkABNxxZ2v/k+FBNsvn8CXefvyEm3OaMks1s+MBxIQa7KnNQKBgFwX +HUo+GiN/+bPCf6P8yFa4J8qI+HEF0SPkZ9cWWx5QzP2M1FZNie++1nce7DcYbBo0 +yh9lyn8W/H328AzDFckS2c5DEY1HtSQPRP3S+AWB5Y7U54h1GMV2L88q6ExWzb60 +T10aeE9b9v+NydmniC5UatTPQIMbht8Tp/u18TAVAoGBAJphAfiqWWV2M5aBCSXq +WxLZ71AJ0PZBmRa/9iwtccwXQpMcW6wHK3YSQxci+sB97TElRa3/onlVSpohrUtg +VCvCwfSHX1LmrfWNSkoJZwCQt+YYuMqW86K0tzLzI1EMjIH9LgQvB6RR26PZQs+E +jr1ZvRc+wPTq6sxCF1h9ZAfN +-----END PRIVATE KEY----- +""", #11 +] + +# To disable the pre-computed tub certs, uncomment this line. +# SYSTEM_TEST_CERTS = [] + + +def flush_but_dont_ignore(res): + d = flushEventualQueue() + def _done(ignored): + return res + d.addCallback(_done) + return d + + +def _render_config(config): + """ + Convert a ``dict`` of ``dict`` of ``unicode`` to an ini-format string. + """ + return u"\n\n".join(list( + _render_config_section(k, v) + for (k, v) + in config.items() + )) + +def _render_config_section(heading, values): + """ + Convert a ``unicode`` heading and a ``dict`` of ``unicode`` to an ini-format + section as ``unicode``. + """ + return u"[{}]\n{}\n".format( + heading, _render_section_values(values) + ) + +def _render_section_values(values): + """ + Convert a ``dict`` of ``unicode`` to the body of an ini-format section as + ``unicode``. + """ + return u"\n".join(list( + u"{} = {}".format(k, v) + for (k, v) + in sorted(values.items()) + )) + + +class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin): + + def setUp(self): + self.port_assigner = SameProcessStreamEndpointAssigner() + self.port_assigner.setUp() + self.addCleanup(self.port_assigner.tearDown) + + self.sparent = service.MultiService() + self.sparent.startService() + + def tearDown(self): + log.msg("shutting down SystemTest services") + d = self.sparent.stopService() + d.addBoth(flush_but_dont_ignore) + return d + + def getdir(self, subdir): + return os.path.join(self.basedir, subdir) + + def add_service(self, s): + s.setServiceParent(self.sparent) + return s + + def _create_introducer(self): + """ + :returns: (via Deferred) an Introducer instance + """ + iv_dir = self.getdir("introducer") + if not os.path.isdir(iv_dir): + _, port_endpoint = self.port_assigner.assign(reactor) + introducer_config = ( + u"[node]\n" + u"nickname = introducer \N{BLACK SMILING FACE}\n" + + u"web.port = {}\n".format(port_endpoint) + ).encode("utf-8") + + fileutil.make_dirs(iv_dir) + fileutil.write( + os.path.join(iv_dir, 'tahoe.cfg'), + introducer_config, + ) + if SYSTEM_TEST_CERTS: + os.mkdir(os.path.join(iv_dir, "private")) + f = open(os.path.join(iv_dir, "private", "node.pem"), "w") + f.write(SYSTEM_TEST_CERTS[0]) + f.close() + return create_introducer(basedir=iv_dir) + + def _get_introducer_web(self): + with open(os.path.join(self.getdir("introducer"), "node.url"), "r") as f: + return f.read().strip() + + @inlineCallbacks + def set_up_nodes(self, NUMCLIENTS=5): + """ + Create an introducer and ``NUMCLIENTS`` client nodes pointed at it. All + of the nodes are running in this process. + + As a side-effect, set: + + * ``numclients`` to ``NUMCLIENTS`` + * ``introducer`` to the ``_IntroducerNode`` instance + * ``introweb_url`` to the introducer's HTTP API endpoint. + + :param int NUMCLIENTS: The number of client nodes to create. + + :return: A ``Deferred`` that fires when the nodes have connected to + each other. + """ + self.numclients = NUMCLIENTS + + self.introducer = yield self._create_introducer() + self.add_service(self.introducer) + self.introweb_url = self._get_introducer_web() + yield self._set_up_client_nodes() + + @inlineCallbacks + def _set_up_client_nodes(self): + q = self.introducer + self.introducer_furl = q.introducer_url + self.clients = [] + basedirs = [] + for i in range(self.numclients): + basedirs.append((yield self._set_up_client_node(i))) + + # start clients[0], wait for it's tub to be ready (at which point it + # will have registered the helper furl). + c = yield client.create_client(basedirs[0]) + c.setServiceParent(self.sparent) + self.clients.append(c) + c.set_default_mutable_keysize(TEST_RSA_KEY_SIZE) + + with open(os.path.join(basedirs[0],"private","helper.furl"), "r") as f: + helper_furl = f.read() + + self.helper_furl = helper_furl + if self.numclients >= 4: + with open(os.path.join(basedirs[3], 'tahoe.cfg'), 'a+') as f: + f.write( + "[client]\n" + "helper.furl = {}\n".format(helper_furl) + ) + + # this starts the rest of the clients + for i in range(1, self.numclients): + c = yield client.create_client(basedirs[i]) + c.setServiceParent(self.sparent) + self.clients.append(c) + c.set_default_mutable_keysize(TEST_RSA_KEY_SIZE) + log.msg("STARTING") + yield self.wait_for_connections() + log.msg("CONNECTED") + # now find out where the web port was + self.webish_url = self.clients[0].getServiceNamed("webish").getURL() + if self.numclients >=4: + # and the helper-using webport + self.helper_webish_url = self.clients[3].getServiceNamed("webish").getURL() + + def _generate_config(self, which, basedir): + config = {} + + except1 = set(range(self.numclients)) - {1} + feature_matrix = { + ("client", "nickname"): except1, + + # client 1 has to auto-assign an address. + ("node", "tub.port"): except1, + ("node", "tub.location"): except1, + + # client 0 runs a webserver and a helper + # client 3 runs a webserver but no helper + ("node", "web.port"): {0, 3}, + ("node", "timeout.keepalive"): {0}, + ("node", "timeout.disconnect"): {3}, + + ("helper", "enabled"): {0}, + } + + def setconf(config, which, section, feature, value): + if which in feature_matrix.get((section, feature), {which}): + config.setdefault(section, {})[feature] = value + + setnode = partial(setconf, config, which, "node") + sethelper = partial(setconf, config, which, "helper") + + setnode("nickname", u"client %d \N{BLACK SMILING FACE}" % (which,)) + + tub_location_hint, tub_port_endpoint = self.port_assigner.assign(reactor) + setnode("tub.port", tub_port_endpoint) + setnode("tub.location", tub_location_hint) + + _, web_port_endpoint = self.port_assigner.assign(reactor) + setnode("web.port", web_port_endpoint) + setnode("timeout.keepalive", "600") + setnode("timeout.disconnect", "1800") + + sethelper("enabled", "True") + + iyaml = ("introducers:\n" + " petname2:\n" + " furl: %s\n") % self.introducer_furl + iyaml_fn = os.path.join(basedir, "private", "introducers.yaml") + fileutil.write(iyaml_fn, iyaml) + + return _render_config(config) + + def _set_up_client_node(self, which): + basedir = self.getdir("client%d" % (which,)) + fileutil.make_dirs(os.path.join(basedir, "private")) + if len(SYSTEM_TEST_CERTS) > (which + 1): + f = open(os.path.join(basedir, "private", "node.pem"), "w") + f.write(SYSTEM_TEST_CERTS[which + 1]) + f.close() + config = self._generate_config(which, basedir) + fileutil.write(os.path.join(basedir, 'tahoe.cfg'), config) + return basedir + + def bounce_client(self, num): + c = self.clients[num] + d = c.disownServiceParent() + # I think windows requires a moment to let the connection really stop + # and the port number made available for re-use. TODO: examine the + # behavior, see if this is really the problem, see if we can do + # better than blindly waiting for a second. + d.addCallback(self.stall, 1.0) + + @defer.inlineCallbacks + def _stopped(res): + new_c = yield client.create_client(self.getdir("client%d" % num)) + self.clients[num] = new_c + new_c.set_default_mutable_keysize(TEST_RSA_KEY_SIZE) + new_c.setServiceParent(self.sparent) + d.addCallback(_stopped) + d.addCallback(lambda res: self.wait_for_connections()) + def _maybe_get_webport(res): + if num == 0: + # now find out where the web port was + self.webish_url = self.clients[0].getServiceNamed("webish").getURL() + d.addCallback(_maybe_get_webport) + return d + + @defer.inlineCallbacks + def add_extra_node(self, client_num, helper_furl=None, + add_to_sparent=False): + # usually this node is *not* parented to our self.sparent, so we can + # shut it down separately from the rest, to exercise the + # connection-lost code + basedir = FilePath(self.getdir("client%d" % client_num)) + basedir.makedirs() + config = "[client]\n" + if helper_furl: + config += "helper.furl = %s\n" % helper_furl + basedir.child("tahoe.cfg").setContent(config.encode("utf-8")) + private = basedir.child("private") + private.makedirs() + write_introducer( + basedir, + "default", + self.introducer_furl, + ) + + c = yield client.create_client(basedir.path) + self.clients.append(c) + c.set_default_mutable_keysize(TEST_RSA_KEY_SIZE) + self.numclients += 1 + if add_to_sparent: + c.setServiceParent(self.sparent) + else: + c.startService() + yield self.wait_for_connections() + defer.returnValue(c) + + def _check_connections(self): + for i, c in enumerate(self.clients): + if not c.connected_to_introducer(): + log.msg("%s not connected to introducer yet" % (i,)) + return False + sb = c.get_storage_broker() + connected_servers = sb.get_connected_servers() + connected_names = sorted(list( + connected.get_nickname() + for connected + in sb.get_known_servers() + if connected.is_connected() + )) + if len(connected_servers) != self.numclients: + wanted = sorted(list( + client.nickname + for client + in self.clients + )) + log.msg( + "client %s storage broker connected to %s, missing %s" % ( + i, + connected_names, + set(wanted) - set(connected_names), + ) + ) + return False + log.msg("client %s storage broker connected to %s, happy" % ( + i, connected_names, + )) + up = c.getServiceNamed("uploader") + if up._helper_furl and not up._helper: + log.msg("Helper fURL but no helper") + return False + return True + + def wait_for_connections(self, ignored=None): + return self.poll(self._check_connections, timeout=200) diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index 902045bdb..1c1383a2d 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -27,7 +27,7 @@ from twisted.internet.defer import inlineCallbacks from foolscap.api import Referenceable from allmydata.interfaces import IStorageServer -from .test_system import SystemTestMixin +from .common_system import SystemTestMixin from .common import AsyncTestCase diff --git a/src/allmydata/test/test_system.py b/src/allmydata/test/test_system.py index 627b6ef29..65ce06328 100644 --- a/src/allmydata/test/test_system.py +++ b/src/allmydata/test/test_system.py @@ -15,24 +15,19 @@ from past.builtins import chr as byteschr, long from six import ensure_text, ensure_str import os, re, sys, time, json -from functools import partial from bs4 import BeautifulSoup -from twisted.internet import reactor from twisted.trial import unittest from twisted.internet import defer -from twisted.internet.defer import inlineCallbacks -from twisted.application import service -from allmydata import client, uri -from allmydata.introducer.server import create_introducer +from allmydata import uri from allmydata.storage.mutable import MutableShareFile from allmydata.storage.server import si_a2b from allmydata.immutable import offloaded, upload from allmydata.immutable.literal import LiteralFileNode from allmydata.immutable.filenode import ImmutableFileNode -from allmydata.util import idlib, mathutil, pollmixin, fileutil +from allmydata.util import idlib, mathutil from allmydata.util import log, base32 from allmydata.util.encodingutil import quote_output, unicode_to_argv from allmydata.util.fileutil import abspath_expanduser_unicode @@ -45,30 +40,19 @@ from allmydata.mutable import layout as mutable_layout from allmydata.mutable.publish import MutableData from allmydata.scripts.runner import PYTHON_3_WARNING -from foolscap.api import DeadReferenceError, fireEventually, flushEventualQueue +from foolscap.api import DeadReferenceError, fireEventually from twisted.python.failure import Failure -from twisted.python.filepath import ( - FilePath, -) from twisted.internet.utils import ( getProcessOutputAndValue, ) -from .common import ( - TEST_RSA_KEY_SIZE, - SameProcessStreamEndpointAssigner, -) from .common_web import do_http as do_http_bytes, Error from .web.common import ( assert_soup_has_tag_with_attributes ) - -# TODO: move this to common or common_util -from . import common_util as testutil +from .common_system import SystemTestMixin from .common_util import run_cli_unicode -from ..scripts.common import ( - write_introducer, -) + class RunBinTahoeMixin(object): def run_bintahoe(self, args, stdin=None, python_options=[], env=None): @@ -117,874 +101,6 @@ This is some data to publish to the remote grid.., which needs to be large enough to not fit inside a LIT uri. """ -# our system test uses the same Tub certificates each time, to avoid the -# overhead of key generation -SYSTEM_TEST_CERTS = [ -"""-----BEGIN CERTIFICATE----- -MIICojCCAYoCAQEwDQYJKoZIhvcNAQELBQAwFzEVMBMGA1UEAwwMbmV3cGJfdGhp -bmd5MB4XDTIwMDEwMjAxNDAzM1oXDTIxMDEwMTAxNDAzM1owFzEVMBMGA1UEAwwM -bmV3cGJfdGhpbmd5MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA1iNV -z07PYwZwucl87QlL2TFZvDxD4flZ/p3BZE3DCT5Efn9w2NT4sHXL1e+R/qsDFuNG -bw1y1TRM0DGK6Wr0XRT2mLQULNgB8y/HrhcSdONsYRyWdj+LimyECKjwh0iSkApv -Yj/7IOuq6dOoh67YXPdf75OHLShm4+8q8fuwhBL+nuuO4NhZDJKupYHcnuCkcF88 -LN77HKrrgbpyVmeghUkwJMLeJCewvYVlambgWRiuGGexFgAm6laS3rWetOcdm9eg -FoA9PKNN6xvPatbj99MPoLpBbzsI64M0yT/wTSw1pj/Nom3rwfMa2OH8Kk7c8R/r -U3xj4ZY1DTlGERvejQIDAQABMA0GCSqGSIb3DQEBCwUAA4IBAQAwyQjQ3ZgtJ3JW -r3/EPdqSUBamTfXIpOh9rXmRjPpbe+MvenqIzl4q+GnkL5mdEb1e1hdKQZgFQ5Q5 -tbcNIz6h5C07KaNtbqhZCx5c/RUEH87VeXuAuOqZHbZWJ18q0tnk+YgWER2TOkgE -RI2AslcsJBt88UUOjHX6/7J3KjPFaAjW1QV3TTsHxk14aYDYJwPdz+ijchgbOPQ0 -i+ilhzcB+qQnOC1s4xQSFo+zblTO7EgqM9KpupYfOVFh46P1Mak2W8EDvhz0livl -OROXJ6nR/13lmQdfVX6T45d+ITBwtmW2nGAh3oI3JlArGKHaW+7qnuHR72q9FSES -cEYA/wmk ------END CERTIFICATE----- ------BEGIN PRIVATE KEY----- -MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDWI1XPTs9jBnC5 -yXztCUvZMVm8PEPh+Vn+ncFkTcMJPkR+f3DY1PiwdcvV75H+qwMW40ZvDXLVNEzQ -MYrpavRdFPaYtBQs2AHzL8euFxJ042xhHJZ2P4uKbIQIqPCHSJKQCm9iP/sg66rp -06iHrthc91/vk4ctKGbj7yrx+7CEEv6e647g2FkMkq6lgdye4KRwXzws3vscquuB -unJWZ6CFSTAkwt4kJ7C9hWVqZuBZGK4YZ7EWACbqVpLetZ605x2b16AWgD08o03r -G89q1uP30w+gukFvOwjrgzTJP/BNLDWmP82ibevB8xrY4fwqTtzxH+tTfGPhljUN -OUYRG96NAgMBAAECggEAJ5xztBx0+nFnisZ9yG8uy6d4XPyc5gE1J4dRDdfgmyYc -j3XNjx6ePi4cHZ/qVryVnrc+AS7wrgW1q9FuS81QFKPbFdZB4SW3/p85BbgY3uxu -0Ovz3T3V9y4polx12eCP0/tKLVd+gdF2VTik9Sxfs5rC8VNN7wmJNuK4A/k15sgy -BIu/R8NlMNGQySNhtccp+dzB8uTyKx5zFZhVvnAK/3YX9BC2V4QBW9JxO4S8N0/9 -48e9Sw/fGCfQ/EFPKGCvTvfuRqJ+4t5k10FygXJ+s+y70ifYi+aSsjJBuranbLJp -g5TwhuKnTWs8Nth3YRLbcJL4VBIOehjAWy8pDMMtlQKBgQD0O8cHb8cOTGW0BijC -NDofhA2GooQUUR3WL324PXWZq0DXuBDQhJVBKWO3AYonivhhd/qWO8lea9MEmU41 -nKZ7maS4B8AJLJC08P8GL1uCIE/ezEXEi9JwC1zJiyl595Ap4lSAozH0DwjNvmGL -5mIdYg0BliqFXbloNJkNlb7INwKBgQDgdGEIWXc5Y1ncWNs6iDIV/t2MlL8vLrP0 -hpkl/QiMndOQyD6JBo0+ZqvOQTSS4NTSxBROjPxvFbEJ3eH8Pmn8gHOf46fzP1OJ -wlYv0gYzkN4FE/tN6JnO2u9pN0euyyZLM1fnEcrMWColMN8JlWjtA7Gbxm8lkfa4 -3vicaJtlWwKBgQCQYL4ZgVR0+Wit8W4qz+EEPHYafvwBXqp6sXxqa7qXawtb+q3F -9nqdGLCfwMNA+QA37ksugI1byfXmpBH902r/aiZbvAkj4zpwHH9F0r0PwbY1iSA9 -PkLahX0Gj8OnHFgWynsVyGOBWVnk9oSHxVt+7zWtGG5uhKdUGLPZugocJQKBgB61 -7bzduOFiRZ5PjhdxISE/UQL2Kz6Cbl7rt7Kp72yF/7eUnnHTMqoyFBnRdCcQmi4I -ZBrnUXbFigamlFAWHhxNWwSqeoVeychUjcRXQT/291nMhRsA02KpNA66YJV6+E9b -xBA6r/vLqGCUUkAWcFfVpIyC1xxV32MmJvAHpBN3AoGAPF3MUFiO0iKNZfst6Tm3 -rzrldLawDo98DRZ7Yb2kWlWZYqUk/Nvryvo2cns75WGSMDYVbbRp+BY7kZmNYa9K -iQzKDL54ZRu6V+getJdeAO8yXoCmnZKxt5OHvOSrQMfAmFKSwLwxBbZBfXEyuune -yfusXLtCgajpreoVIa0xWdQ= ------END PRIVATE KEY----- -""", # 0 -"""-----BEGIN CERTIFICATE----- -MIICojCCAYoCAQEwDQYJKoZIhvcNAQELBQAwFzEVMBMGA1UEAwwMbmV3cGJfdGhp -bmd5MB4XDTIwMDEwMjAxNDAzM1oXDTIxMDEwMTAxNDAzM1owFzEVMBMGA1UEAwwM -bmV3cGJfdGhpbmd5MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEApDzW -4ZBeK9w4xpRaed6lXzeCO0Xmr3f0ynbueSdiZ89FWoAMgK+SiBIOViYV6hfm0Wah -lemSNzFGx5LvDSg2uwSqEP23DeM9O/SQPgIAiLeeEsYZJcgg2jz92YfFEaahsGdI -6qSP4XI2/5dgKRpPOYDGyw6R5PQR6w22Xq1WD1jBvImk/k09I9jHRn40pYbaJzbg -U2aIjvOruo2kqe4f6iDqE0piYimAZJUvemu1UoyV5NG590hGkDuWsMD77+d2FxCj -9Nzb+iuuG3ksnanHPyXi1hQmzp5OmzVWaevCHinNjWgsuSuLGO9H2SLf3wwp2UCs -EpKtzoKrnZdEg/anNwIDAQABMA0GCSqGSIb3DQEBCwUAA4IBAQChxtr67o1aZZMJ -A6gESPtFjZLw6wG0j50JsrWKLvoXVts1ToJ9u2nx01aFKjBwb4Yg+vdJfDgIIAEm -jS56h6H2DfJlkTWHmi8Vx1wuusWnrNwYMI53tdlRIpD2+Ne7yeoLQZcVN2wuPmxD -Mbksg4AI4csmbkU/NPX5DtMy4EzM/pFvIcxNIVRUMVTFzn5zxhKfhyPqrMI4fxw1 -UhUbEKO+QgIqTNp/dZ0lTbFs5HJQn6yirWyyvQKBPmaaK+pKd0RST/T38OU2oJ/J -LojRs7ugCJ+bxJqegmQrdcVqZZGbpYeK4O/5eIn8KOlgh0nUza1MyjJJemgBBWf7 -HoXB8Fge ------END CERTIFICATE----- ------BEGIN PRIVATE KEY----- -MIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQCkPNbhkF4r3DjG -lFp53qVfN4I7Reavd/TKdu55J2Jnz0VagAyAr5KIEg5WJhXqF+bRZqGV6ZI3MUbH -ku8NKDa7BKoQ/bcN4z079JA+AgCIt54SxhklyCDaPP3Zh8URpqGwZ0jqpI/hcjb/ -l2ApGk85gMbLDpHk9BHrDbZerVYPWMG8iaT+TT0j2MdGfjSlhtonNuBTZoiO86u6 -jaSp7h/qIOoTSmJiKYBklS96a7VSjJXk0bn3SEaQO5awwPvv53YXEKP03Nv6K64b -eSydqcc/JeLWFCbOnk6bNVZp68IeKc2NaCy5K4sY70fZIt/fDCnZQKwSkq3Ogqud -l0SD9qc3AgMBAAECggEBAIu55uaIOFYASZ1IYaEFNpRHWVisI5Js76nAfSo9w46l -3E8eWYSx2mxBUEkipco/A3RraFVuHaMvHRR1gUMkT0vUsAs8jxwVk+cKLh1S/rlR -3f4C4yotlSWWdjE3PQXDShQWCwb1ciNPVFMmqfzOEVDOqlHe12h97TCYverWdT0f -3LZICLQsZd1WPKnPNXrsRRDCBuRLapdg+M0oJ+y6IiCdm+qM7Qvaoef6hlvm5ECz -LCM92db5BKTuPOQXMx2J8mjaBgU3aHxRV08IFgs7mI6q0t0FM7LlytIAJq1Hg5QU -36zDKo8tblkPijWZWlqlZCnlarrd3Ar/BiLEiuOGDMECgYEA1GOp0KHy0mbJeN13 -+TDsgP7zhmqFcuJREu2xziNJPK2S06NfGYE8vuVqBGzBroLTQ3dK7rOJs9C6IjCE -mH7ZeHzfcKohpZnl443vHMSpgdh/bXTEO1aQZNbJ2hLYs8ie/VqqHR0u6YtpUqZL -LgaUA0U8GnlsO55B8kyCelckmDkCgYEAxfYQMPEEzg1tg2neqEfyoeY0qQTEJTeh -CPMztowSJpIyF1rQH6TaG0ZchkiAkw3W58RVDfvK72TuVlC5Kz00C2/uPnrqm0dX -iMPeML5rFlG3VGCrSTnAPI+az6P65q8zodqcTtA8xoxgPOlc/lINOxiTEMxLyeGF -8GyP+sCM2u8CgYEAvMBR05OJnEky9hJEpBZBqSZrQGL8dCwDh0HtCdi8JovPd/yx -8JW1aaWywXnx6uhjXoru8hJm54IxWV8rB+d716OKY7MfMfACqWejQDratgW0wY7L -MjztGGD2hLLJGYXLHjfsBPHBllaKZKRbHe1Er19hWdndQWKVEwPB1X4KjKkCgYEA -nWHmN3K2djbYtRyLR1CEBtDlVuaSJmCWp23q1BuCJqYeKtEpG69NM1f6IUws5Dyh -eXtuf4KKMU8V6QueW1D6OomPaJ8CO9c5MWM/F5ObwY/P58Y/ByVhvwQQeToONC5g -JzKNCF+nodZigKqrIwoKuMvtx/IT4vloKd+1jA5fLYMCgYBoT3HLCyATVdDSt1TZ -SbEDoLSYt23KRjQV93+INP949dYCagtgh/kTzxBopw5FljISLfdYizIRo2AzhhfP -WWpILlnt19kD+sNirJVqxJacfEZsu5baWTedI/yrCuVsAs/s3/EEY6q0Qywknxtp -Fwh1/8y5t14ib5fxOVhi8X1nEA== ------END PRIVATE KEY----- -""", # 1 -"""-----BEGIN CERTIFICATE----- -MIICojCCAYoCAQEwDQYJKoZIhvcNAQELBQAwFzEVMBMGA1UEAwwMbmV3cGJfdGhp -bmd5MB4XDTIwMDEwMjAxNDAzM1oXDTIxMDEwMTAxNDAzM1owFzEVMBMGA1UEAwwM -bmV3cGJfdGhpbmd5MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAwMTn -hXnpKHGAir3WYbOxefVrMA07OZNAsNa29nBwLA+NVIJNUFgquibMj7QYo8+M45oY -6LKr4yRcBryZVvyxfdr92xp8+kLeVApk2WLjkdBTRagHh9qdrY0hQmagCBN6/hLG -Xug8VksQUdhX3vu6ZyMvTLfKRkDOMRVkRGRGg/dOcvom7zpqMCGYenMG2FStr6UV -3s3dlCSZZTdTX5Uoq6yfUUJE3nITGKjpnpJKqIs3PWCIxdj7INIcjJKvIdUcavIV -2hEhh60A8ltmtdpQAXVBE+U7aZgS1fGAWS2A0a3UwuP2pkQp6OyKCUVHpZatbl9F -ahDN2QBzegv/rdJ1zwIDAQABMA0GCSqGSIb3DQEBCwUAA4IBAQAl4OQZ+FB9ZSUv -FL/KwLNt+ONU8Sve/xiX+8vKAvgKm2FrjyK+AZPwibnu+FSt2G4ndZBx4Wvpe5V+ -gCsbzSXlh9cDn2SRXyprt2l/8Fj4eUMaThmLKOK200/N/s2SpmBtnuflBrhNaJpw -DEi2KEPuXsgvkuVzXN06j75cUHwn5LeWDAh0RalkVuGbEWBoFx9Hq8WECdlCy0YS -y09+yO01qz70y88C2rPThKw8kP4bX8aFZbvsnRHsLu/8nEQNlrELcfBarPVHjJ/9 -imxOdymJkV152V58voiXP/PwXhynctQbF7e+0UZ+XEGdbAbZA0BMl7z+b09Z+jF2 -afm4mVox ------END CERTIFICATE----- ------BEGIN PRIVATE KEY----- -MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDAxOeFeekocYCK -vdZhs7F59WswDTs5k0Cw1rb2cHAsD41Ugk1QWCq6JsyPtBijz4zjmhjosqvjJFwG -vJlW/LF92v3bGnz6Qt5UCmTZYuOR0FNFqAeH2p2tjSFCZqAIE3r+EsZe6DxWSxBR -2Ffe+7pnIy9Mt8pGQM4xFWREZEaD905y+ibvOmowIZh6cwbYVK2vpRXezd2UJJll -N1NflSirrJ9RQkTechMYqOmekkqoizc9YIjF2Psg0hyMkq8h1Rxq8hXaESGHrQDy -W2a12lABdUET5TtpmBLV8YBZLYDRrdTC4/amRCno7IoJRUellq1uX0VqEM3ZAHN6 -C/+t0nXPAgMBAAECggEAF+2ZK4lZdsq4AQDVhqUuh4v+NSW/T0NHCWxto6OLWPzJ -N09BV5LKIvdD9yaM1HCj9XCgXOooyfYuciuhARo20f+H+VWNY+c+/8GWiSFsTCJG -4+Oao7NwVSWqljp07Ou2Hamo9AjxzGhe6znmlmg62CiW63f45MWQkqksHA0yb5jg -/onJ2//I+OI+aTKNfjt1G6h2x7oxeGTU1jJ0Hb2xSh+Mpqx9NDfb/KZyOndhSG5N -xRVosQ6uV+9mqHxTTwTZurTG31uhZzarkMuqxhcHS94ub7berEc/OlqvbyMKNZ3A -lzuvq0NBZhEUhAVgORAIS17r/q2BvyG4u5LFbG2p0QKBgQDeyyOl+A7xc4lPE2OL -Z3KHJPP4RuUnHnWFC+bNdr5Ag8K7jcjZIcasyUom9rOR0Fpuw9wmXpp3+6fyp9bJ -y6Bi5VioR0ZFP5X+nXxIN3yvgypu6AZvkhHrEFer+heGHxPlbwNKCKMbPzDZPBTZ -vlC7g7xUUcpNmGhrOKr3Qq5FlwKBgQDdgCmRvsHUyzicn8TI3IJBAOcaQG0Yr/R2 -FzBqNfHHx7fUZlJfKJsnu9R9VRZmBi4B7MA2xcvz4QrdZWEtY8uoYp8TAGILfW1u -CP4ZHrzfDo/67Uzk2uTMTd0+JOqSm/HiVNguRPvC8EWBoFls+h129GKThMvKR1hP -1oarfAGIiQKBgQCIMAq5gHm59JMhqEt4QqMKo3cS9FtNX1wdGRpbzFMd4q0dstzs -ha4Jnv3Z9YHtBzzQap9fQQMRht6yARDVx8hhy6o3K2J0IBtTSfdXubtZGkfNBb4x -Y0vaseG1uam5jbO+0u5iygbSN/1nPUfNln2JMkzkCh8s8ZYavMgdX0BiPwKBgChR -QL/Hog5yoy5XIoGRKaBdYrNzkKgStwObuvNKOGUt5DckHNA3Wu6DkOzzRO1zKIKv -LlmJ7VLJ3qln36VcaeCPevcBddczkGyb9GxsHOLZCroY4YsykLzjW2cJXy0qd3/E -A8mAQvc7ttsebciZSi2x1BOX82QxUlDN8ptaKglJAoGBAMnLN1TQB0xtWYDPGcGV -2IvgX7OTRRlMVrTvIOvP5Julux9z1r0x0cesl/jaXupsBUlLLicPyBMSBJrXlr24 -mrgkodk4TdqO1VtBCZBqak97DHVezstMrbpCGlUD5jBnsHVRLERvS09QlGhqMeNL -jpNQbWH9VhutzbvpYquKrhvK ------END PRIVATE KEY----- -""", # 2 -"""-----BEGIN CERTIFICATE----- -MIICojCCAYoCAQEwDQYJKoZIhvcNAQELBQAwFzEVMBMGA1UEAwwMbmV3cGJfdGhp -bmd5MB4XDTIwMDEwMjAxNDAzM1oXDTIxMDEwMTAxNDAzM1owFzEVMBMGA1UEAwwM -bmV3cGJfdGhpbmd5MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAypqi -YTni3s60Uo8vgGcFvjWWkB5CD9Fx9pW/2KcxRJ/u137Y+BG8qWMA4lgII3ZIuvo4 -6rLDiXnAnDZqUtrvZ90O/gH6RyQqX3AI4EwPvCnRIIe0okRcxnxYBL/LfBY54xuv -46JRYZP4c9IImqQH9QVo2/egtEzcpbmT/mfhpf6NGQWC3Xps2BqDT2SV/DrX/wPA -8P1atE1AxNp8ENxK/cjFAteEyDZOsDSa757ZHKAdM7L8rZ1Fd2xAA1Dq7IyYpTNE -IX72xytWxllcNvSUPLT+oicsSZBadc/p3moc3tR/rNdgrHKybedadru/f9Gwpa+v -0sllZlEcVPSYddAzWwIDAQABMA0GCSqGSIb3DQEBCwUAA4IBAQCmk60Nj5FPvemx -DSSQjJPyJoIDpTxQ4luSzIq4hPwlUXw7dqrvHyCWgn2YVe9xZsGrT/+n376ecmgu -sw4s4qVhR9bzKkTMewjC2wUooTA5v9HYsNWZy3Ah7hHPbDHlMADYobjB5/XolNUP -bCM9xALEdM9DxpC4vjUZexlRKmjww9QKE22jIM+bqsK0zqDSq+zHpfHNGGcS3vva -OvI6FPc1fAr3pZpVzevMSN2zufIJwjL4FT5/uzwOCaSCwgR1ztD5CSbQLTLlwIsX -S7h2WF9078XumeRjKejdjEjyH4abKRq8+5LVLcjKEpg7OvktuRpPoGPCEToaAzuv -h+RSQwwY ------END CERTIFICATE----- ------BEGIN PRIVATE KEY----- -MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDKmqJhOeLezrRS -jy+AZwW+NZaQHkIP0XH2lb/YpzFEn+7Xftj4EbypYwDiWAgjdki6+jjqssOJecCc -NmpS2u9n3Q7+AfpHJCpfcAjgTA+8KdEgh7SiRFzGfFgEv8t8FjnjG6/jolFhk/hz -0giapAf1BWjb96C0TNyluZP+Z+Gl/o0ZBYLdemzYGoNPZJX8Otf/A8Dw/Vq0TUDE -2nwQ3Er9yMUC14TINk6wNJrvntkcoB0zsvytnUV3bEADUOrsjJilM0QhfvbHK1bG -WVw29JQ8tP6iJyxJkFp1z+neahze1H+s12CscrJt51p2u79/0bClr6/SyWVmURxU -9Jh10DNbAgMBAAECggEBALv7Q+Rf+C7wrQDZF6LUc9CrGfq4CGVy2IGJKgqT/jOF -DO9nI1rv4hNr55sbQNneWtcZaYvht2mrzNlj57zepDjDM7DcFuLBHIuWgLXT/NmC -FyZOo3vXYBlNr8EgT2XfnXAp9UWJCmc2CtUzsIYC4dsmXMeTd8kyc5tUl4r5ybTf -1g+RTck/IGgqdfzpuTsNl79FW2rP9z111Py6dbqgQzhuSAune9dnLFvZst8dyL8j -FStETMxBM6jrCF1UcKXzG7trDHiCdzJ8WUhx6opN/8OasQGndwpXto6FZuBy/AVP -4kVQNpUXImYcLEpva0MqGRHg+YN+c84C71CMchnF4aECgYEA7J2go4CkCcZNKCy5 -R5XVCqNFYRHjekR+UwH8cnCa7pMKKfP+lTCiBrO2q8zwWwknRMyuycS5g/xbSpg1 -L6hi92CV1YQy1/JhlQedekjejNTTuLOPKf78AFNSfc7axDnes2v4Bvcdp9gsbUIO -10cXh0tOSLE7P9y+yC86KQkFAPECgYEA2zO0M2nvbPHv2jjtymY3pflYm0HzhM/T -kPtue3GxOgbEPsHffBGssShBTE3yCOX3aAONXJucMrSAPL9iwUfgfGx6ADdkwBsA -OjDlkxvTbP/9trE6/lsSPtGpWRdJNHqXN4Hx7gXJizRwG7Ym+oHvIIh53aIjdFoE -HLQLpxObuQsCgYAuMQ99G83qQpYpc6GwAeYXL4yJyK453kky9z5LMQRt8rKXQhS/ -F0FqQYc1vsplW0IZQkQVC5yT0Z4Yz+ICLcM0O9zEVAyA78ZxC42Io9UedSXn9tXK -Awc7IQkHmmxGxm1dZYSEB5X4gFEb+zted3h2ZxMfScohS3zLI70c6a/aYQKBgQCU -phRuxUkrTUpFZ1PCbN0R/ezbpLbaewFTEV7T8b6oxgvxLxI6FdZRcSYO89DNvf2w -GLCVe6VKMWPBTlxPDEostndpjCcTq3vU+nHE+BrBkTvh14BVGzddSFsaYpMvNm8z -ojiJHH2XnCDmefkm6lRacJKL/Tcj4SNmv6YjUEXLDwKBgF8WV9lzez3d/X5dphLy -2S7osRegH99iFanw0v5VK2HqDcYO9A7AD31D9nwX46QVYfgEwa6cHtVCZbpLeJpw -qXnYXe/hUU3yn5ipdNJ0Dm/ZhJPDD8TeqhnRRhxbZmsXs8EzfwB2tcUbASvjb3qA -vAaPlOSU1wXqhAsG9aVs8gtL ------END PRIVATE KEY----- -""", # 3 -"""-----BEGIN CERTIFICATE----- -MIICojCCAYoCAQEwDQYJKoZIhvcNAQELBQAwFzEVMBMGA1UEAwwMbmV3cGJfdGhp -bmd5MB4XDTIwMDEwMjAxNDAzNFoXDTIxMDEwMTAxNDAzNFowFzEVMBMGA1UEAwwM -bmV3cGJfdGhpbmd5MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAzUqQ -M08E7F2ZE99bFHvpsR6LmgIJOOoGMXacTcEUhRF63E6+730FjxER2a30synv9GGS -3G9FstUmfhyimufkbTumri8Novw5CWZQLiE1rmMBI5nPcR2wAzy9z2odR6bfAwms -yyc3IPYg1BEDBPZl0LCQrQRRU/rVOrbCf7IMq+ATazmBg01gXMzq2M953ieorkQX -MsHVR/kyW0Q0yzhYF1OtIqbXxrdiZ+laTLWNqivj/FdegiWPCf8OcqpcpbgEjlDW -gBcC/vre+0E+16nfUV8xHL5jseJMJqfT508OtHxAzp+2D7b54NvYNIvbOAP+F9gj -aXy5mOvjXclK+hNmDwIDAQABMA0GCSqGSIb3DQEBCwUAA4IBAQAjZzTFKG7uoXxm -BPHfQvsKHIB/Cx9zMKj6pLwJzCPHQBzKOMoUen09oq+fb77RM7WvdX0pvFgEXaJW -q/ImooRMo+paf8GOZAuPwdafb2/OGdHZGZ2Cbo/ICGo1wGDCdMvbxTxrDNq1Yae+ -m+2epN2pXAO1rlc7ktRkojM/qi3zXtbLjTs3IoPDXWhYPHdI1ThkneRmvxpzB1rW -2SBqj2snvyI+/3k3RHmldcdOrTlgWQ9hq05jWR8IVtRUFFVn9A+yQC3gnnLIUhwP -HJWwTIPuYW25TuxFxYZXIbnAiluZL0UIjd3IAwxaafvB6uhI7v0K789DKj2vRUkY -E8ptxZH4 ------END CERTIFICATE----- ------BEGIN PRIVATE KEY----- -MIIEwAIBADANBgkqhkiG9w0BAQEFAASCBKowggSmAgEAAoIBAQDNSpAzTwTsXZkT -31sUe+mxHouaAgk46gYxdpxNwRSFEXrcTr7vfQWPERHZrfSzKe/0YZLcb0Wy1SZ+ -HKKa5+RtO6auLw2i/DkJZlAuITWuYwEjmc9xHbADPL3Pah1Hpt8DCazLJzcg9iDU -EQME9mXQsJCtBFFT+tU6tsJ/sgyr4BNrOYGDTWBczOrYz3neJ6iuRBcywdVH+TJb -RDTLOFgXU60iptfGt2Jn6VpMtY2qK+P8V16CJY8J/w5yqlyluASOUNaAFwL++t77 -QT7Xqd9RXzEcvmOx4kwmp9PnTw60fEDOn7YPtvng29g0i9s4A/4X2CNpfLmY6+Nd -yUr6E2YPAgMBAAECggEBAIiL6uQl0AmDrBj6vHMghGzp+0MBza6MgngOA6L4JTTp -ToYQ3pEe4D6rxOq7+QHeiBtNd0ilvn9XpVXGqCVOzrIVNiWvaGubRjjJU9WLA1Ct -y4kpekAr1fIhScMXOsh45ub3XXZ27AVBkM5dTlvTpB8uAd0C/TFVqtR10WLsQ99h -Zm9Jczgs/6InYTssnAaqdeCLAf1LbmO4zwFsJfJOeSGGT6WBwlpHwMAgPhg8OLEu -kVWG7BEJ0hxcODk/es/vce9SN7BSyIzNY+qHcGtsrx/o0eO2Av/Z7ltV4Sz6UN1K -0y0OTiDyT/l62U2OugSN3wQ4xPTwlrWl7ZUHJmvpEaECgYEA+w2JoB2i1OV2JTPl -Y0TKSKcZYdwn7Nwh4fxMAJNJ8UbpPqrZEo37nxqlWNJrY/jKX3wHVk4ESSTaxXgF -UY7yKT0gRuD9+vE0gCbUmJQJTwbceNJUu4XrJ6SBtf72WgmphL+MtyKdwV8XltVl -Yp0hkswGmxl+5+Js6Crh7WznPl8CgYEA0VYtKs2YaSmT1zraY6Fv3AIQZq012vdA -7nVxmQ6jKDdc401OWARmiv0PrZaVNiEJ1YV8KxaPrKTfwhWqxNegmEBgA1FZ66NN -SAm8P9OCbt8alEaVkcATveXTeOCvfpZUO3sqZdDOiYLiLCsokHblkcenK85n0yT6 -CzhTbvzDllECgYEAu9mfVy2Vv5OK2b+BLsw0SDSwa2cegL8eo0fzXqLXOzCCKqAQ -GTAgTSbU/idEr+NjGhtmKg/qaQioogVyhVpenLjeQ+rqYDDHxfRIM3rhlD5gDg/j -0wUbtegEHrgOgcSlEW16zzWZsS2EKxq16BoHGx6K+tcS/FOShg5ASzWnuiUCgYEA -sMz+0tLX8aG7CqHbRyBW8FMR9RY/kRMY1Q1+Bw40wMeZfSSSkYYN8T9wWWT/2rqm -qp7V0zJ34BFUJoDUPPH84fok3Uh9EKZYpAoM4z9JP0jREwBWXMYEJnOQWtwxfFGN -DLumgF2Nwtg3G6TL2s+AbtJYH4hxagQl5woIdYmnyzECgYEAsLASpou16A3uXG5J -+5ZgF2appS9Yfrqfh6TKywMsGG/JuiH3djdYhbJFIRGeHIIDb4XEXOHrg/SFflas -If0IjFRh9WCvQxnoRha3/pKRSc3OEka1MR/ZREK/d/LQEPmsRJVzY6ABKqmPAMDD -5CnG6Hz/rP87BiEKd1+3PGp8GCw= ------END PRIVATE KEY----- -""", # 4 -"""-----BEGIN CERTIFICATE----- -MIICojCCAYoCAQEwDQYJKoZIhvcNAQELBQAwFzEVMBMGA1UEAwwMbmV3cGJfdGhp -bmd5MB4XDTIwMDEwMjAxNDAzNFoXDTIxMDEwMTAxNDAzNFowFzEVMBMGA1UEAwwM -bmV3cGJfdGhpbmd5MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA0sap -75YbbkEL85LFava3FrO1jpgVteQ4NGxxy1Nu9w2hPfMMeCPWjB8UfAwFk+LVPyvW -LAXd1zWL5rGpQ2ytIVQlTraR5EnALA1sMcQYbFz1ISPTYB031bEN/Ch8JWYwCG5A -X2H4D6BC7NgT6YyWDt8vxQnqAisPHQ/OK4ABD15CwkTyPimek2/ufYN2dapg1xhG -IUD96gqetJv9bu0r869s688kADIComsYG+8KKfFN67S3rSHMIpZPuGTtoHGnVO89 -XBm0vNe0UxQkJEGJzZPn0tdec0LTC4GNtTaz5JuCjx/VsJBqrnTnHHjx0wFz8pff -afCimRwA+LCopxPE1QIDAQABMA0GCSqGSIb3DQEBCwUAA4IBAQBOkAnpBb3nY+dG -mKCjiLqSsuEPqpNiBYR+ue/8aVDnOKLKqAyQuyRZttQ7bPpKHaw7pwyCZH8iHnt6 -pMCLCftNSlV2Fa8msRmuf5AiGjUvR1M8VtHWNYE8pedWrJqUgBhF/405B99yd8CT -kQJXKF18LObj7YKNsWRoMkVgqlQzWDMEqbfmy9MhuLx2EZPsTB1L0BHNGGDVBd9o -cpPLUixcc12u+RPMKq8x3KgwsnUf5vX/pCnoGcCy4JahWdDgcZlf0hUKGT7PUem5 -CWW8SMeqSWQX9XpE5Qlm1+W/QXdDXLbbHqDtvBeUy3iFQe3C9RSkp0qdutxkAlFk -f5QHXfJ7 ------END CERTIFICATE----- ------BEGIN PRIVATE KEY----- -MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDSxqnvlhtuQQvz -ksVq9rcWs7WOmBW15Dg0bHHLU273DaE98wx4I9aMHxR8DAWT4tU/K9YsBd3XNYvm -salDbK0hVCVOtpHkScAsDWwxxBhsXPUhI9NgHTfVsQ38KHwlZjAIbkBfYfgPoELs -2BPpjJYO3y/FCeoCKw8dD84rgAEPXkLCRPI+KZ6Tb+59g3Z1qmDXGEYhQP3qCp60 -m/1u7Svzr2zrzyQAMgKiaxgb7wop8U3rtLetIcwilk+4ZO2gcadU7z1cGbS817RT -FCQkQYnNk+fS115zQtMLgY21NrPkm4KPH9WwkGqudOccePHTAXPyl99p8KKZHAD4 -sKinE8TVAgMBAAECggEALU5EotoqJUXYEtAenUJQ0pFoWjE4oXNf3Wzd/O1/MZ19 -ZjqDGKPjbxUTKyLOZB5i5gQ/MhFEwQiifMD9eB+5CyvyJPw7Wc28f/uWoQ/cjBZj -Hm979PHy2X0IW4Y8QTG462b/cUE2t+0j1ZMQnKf6bVHuC7V41mR5CC8oitMl5y5g -34yJmWXlIA0ep/WotLMqvil6DnSM/2V8Ch4SxjnzPpjbe4Kj+woucGNr4UKstZER -8iuHTsR64LjoGktRnnMwZxGZQI7EC428zsliInuWMdXe//w2chLdkirqpSrIQwSZ -3jNWStqBXGYaRg5Z1ilBvHtXxkzDzbAlzRBzqfEwwQKBgQDqYdMRrzHJaXWLdsyU -6jAuNX9tLh7PcicjP93SbPujS6mWcNb+D/au+VhWD+dZQDPRZttXck7wvKY1lw1V -MK0TYI7ydf8h3DFx3Mi6ZD4JVSU1MH233C3fv/FHenDoOvMXXRjUZxaRmuzFJvzt -6QlKIfSvwT+1wrOACNfteXfZUQKBgQDmN3Uuk01qvsETPwtWBp5RNcYhS/zGEQ7o -Q4K+teU453r1v8BGsQrCqulIZ3clMkDru2UroeKn1pzyVAS2AgajgXzfXh3VeZh1 -vHTLP91BBYZTTWggalEN4aAkf9bxX/hA+9Bw/dzZcQW2aNV7WrYuCSvp3SDCMina -anQq/PaSRQKBgHjw23HfnegZI89AENaydQQTFNqolrtiYvGcbgC7vakITMzVEwrr -/9VP0pYuBKmYKGTgF0RrNnKgVX+HnxibUmOSSpCv9GNrdJQVYfpT6XL1XYqxp91s -nrs7FuxUMNiUOoWOw1Yuj4W4lH4y3QaCXgnDtbfPFunaOrdRWOIv8HjRAoGAV3NT -mSitbNIfR69YIAqNky3JIJbb42VRc1tJzCYOd+o+pCF96ZyRCNehnDZpZQDM9n8N -9GAfWEBHCCpwS69DVFL422TGEnSJPJglCZwt8OgnWXd7CW05cvt1OMgzHyekhxLg -4Dse7J5pXBxAlAYmVCB5xPGR4xLpISX1EOtcwr0CgYEA5rA2IUfjZYb4mvFHMKyM -xWZuV9mnl3kg0ULttPeOl3ppwjgRbWpyNgOXl8nVMYzxwT/A+xCPA18P0EcgNAWc -frJqQYg3NMf+f0K1wSaswUSLEVrQOj25OZJNpb21JEiNfEd5DinVVj4BtVc6KSpS -kvjbn2WhEUatc3lPL3V0Fkw= ------END PRIVATE KEY----- -""", # 5 -"""-----BEGIN CERTIFICATE----- -MIICojCCAYoCAQEwDQYJKoZIhvcNAQELBQAwFzEVMBMGA1UEAwwMbmV3cGJfdGhp -bmd5MB4XDTIwMDEwMjAxNTExM1oXDTIxMDEwMTAxNTExM1owFzEVMBMGA1UEAwwM -bmV3cGJfdGhpbmd5MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA1c5y -S9IZHF9MIuwdafzhMkgP37I3RVpHEbpnPwnLFqSWelS5m2eDkwWd5SkfGjrmQ5q0 -PEpqLlh3zHGw9yQjnHS3CCS1PwQ1kmwvpIK3HM5y8GM7ry1zkam8ZR4iX6Y7VG9g -9mhiVVFoVhe1gHeiC/3Mp6XeNuEiD0buM+8qZx9B21I+iwzy4wva7Gw0fJeq9G1c -lq2rhpD1LlIEodimWOi7lOEkNmUiO1SvpdrGdxUDpTgbdg6r5pCGjOXLd74tAQHP -P/LuqRNJDXtwvHtLIVQnW6wjjy4oiWZ8DXOdc9SkepwQLIF5Wh8O7MzF5hrd6Cvw -SOD3EEsJbyycAob6RwIDAQABMA0GCSqGSIb3DQEBCwUAA4IBAQBDNcbKVUyGOAVm -k3iVuzkkkymlTAMm/gsIs6loLJrkSqNg160FdVKJoZFjQtqoqLgLrntdCJ377nZ9 -1i+yzbZsA4DA7nxj0IEdnd7rRYgGLspGqWeKSTROATeT4faLTXenecm0v2Rpxqc7 -dSyeZJXOd2OoUu+Q64hzXCDXC6LNM+xZufxV9qv+8d+CipV6idSQZaUWSVuqFCwD -PT0R4eWfkMMaM8QqtNot/hVCEaKT+9rG0mbpRe/b/qBy5SR0u+XgGEEIV+33L59T -FXY+DpI1Dpt/bJFoUrfj6XohxdTdqYVCn1F8in98TsRcFHyH1xlkS3Y0RIiznc1C -BwAoGZ4B ------END CERTIFICATE----- ------BEGIN PRIVATE KEY----- -MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDVznJL0hkcX0wi -7B1p/OEySA/fsjdFWkcRumc/CcsWpJZ6VLmbZ4OTBZ3lKR8aOuZDmrQ8SmouWHfM -cbD3JCOcdLcIJLU/BDWSbC+kgrccznLwYzuvLXORqbxlHiJfpjtUb2D2aGJVUWhW -F7WAd6IL/cynpd424SIPRu4z7ypnH0HbUj6LDPLjC9rsbDR8l6r0bVyWrauGkPUu -UgSh2KZY6LuU4SQ2ZSI7VK+l2sZ3FQOlOBt2DqvmkIaM5ct3vi0BAc8/8u6pE0kN -e3C8e0shVCdbrCOPLiiJZnwNc51z1KR6nBAsgXlaHw7szMXmGt3oK/BI4PcQSwlv -LJwChvpHAgMBAAECggEBAK0KLeUBgIM++Y7WDCRInzYjrn08bpE5tIU7mO4jDfQg -dw1A3wtQZuOpyxW6B0siWlRis/aLv44M2cBkT3ZmEFBDAhOcKfh7fqQn3RNHG847 -pDi8B4UKwxskBa7NCcLh9eirUA19hABLJ6dt/t6fdE5CNc2FZ+iAoyE8JfNwYKAd -6Fa3HqUBPNWt8ryj4ftgpMNBdfmLugEM4N20SXJA28hOq2lUcwNKQQ1xQrovl0ig -iMbMWytV4gUPKC9Wra66OYIkk/K8teiUNIYA4JwAUVTs1NEWoyfwUTz1onutCkMl -5vY7JAqRoDWoSUX6FI+IHUdyqPAMdOMhC37gjrxoo2ECgYEA7trDMu6xsOwEckDh -iz148kejMlnTTuCNetOFBw3njFgxISx0PrDLWmJmnHMxPv9AAjXYb2+UCCm3fj6Q -OB8o4ZJm0n504qbFHcb2aI22U5hZ99ERvqx8WBnJ2RarIBmg06y0ktxq8gFR2qxF -0hWAOcDn1DWQ8QI0XBiFFcJTGtcCgYEA5SdlIXRnVZDKi5YufMAORG9i74dXUi0Y -02UoVxJ+q8VFu+TT8wrC5UQehG3gX+79Cz7hthhDqOSCv6zTyE4Evb6vf9OLgnVe -E5iLF033zCxLSS9MgiZ+jTO+wK3RsapXDtGcSEk2P82Pj5seNf4Ei1GNCRlm1DbX -71wlikprHhECgYABqmLcExAIJM0vIsav2uDiB5/atQelMCmsZpcx4mXv85l8GrxA -x6jTW4ZNpvv77Xm7yjZVKJkGqYvPBI6q5YS6dfPjmeAkyHbtazrCpeJUmOZftQSD -qN5BGwTuT5sn4SXe9ABaWdEhGONCPBtMiLvZK0AymaEGHTbSQZWD/lPoBwKBgGhk -qg2zmd/BNoSgxkzOsbE7jTbR0VX+dXDYhKgmJM7b8AjJFkWCgYcwoTZzV+RcW6rj -2q+6HhizAV2QvmpiIIbQd+Mj3EpybYk/1R2ox1qcUy/j/FbOcpihGiVtCjqF/2Mg -2rGTqMMoQl6JrBmsvyU44adjixTiZz0EHZYCkQoBAoGBAMRdmoR4mgIIWFPgSNDM -ISLJxKvSFPYDLyAepLfo38NzKfPB/XuZrcOoMEWRBnLl6dNN0msuzXnPRcn1gc1t -TG7db+hivAyUoRkIW3dB8pRj9dDUqO9OohjKsJxJaQCyH5vPkQFSLbTIgWrHhU+3 -oSPiK/YngDV1AOmPDH7i62po ------END PRIVATE KEY----- -""", #6 -"""-----BEGIN CERTIFICATE----- -MIICojCCAYoCAQEwDQYJKoZIhvcNAQELBQAwFzEVMBMGA1UEAwwMbmV3cGJfdGhp -bmd5MB4XDTIwMDEwMjAxNTExMloXDTIxMDEwMTAxNTExMlowFzEVMBMGA1UEAwwM -bmV3cGJfdGhpbmd5MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAojGu -fQaTVT9DJWJ/zogGfrryEJXYVy9c441O5MrLlRx7nCIWIUs2NEhHDJdqJjYOTdmk -K98VhdMpDPZwxjgvvZrh43lStBRIW3zZxv747rSl2VtpSqD/6UNWJe5u4SR7oga4 -JfITOKHg/+ASxnOxp/iu6oT6jBL6T7KSPh6Rf2+it2rsjhktRreFDJ2hyroNq1w4 -ZVNCcNPgUIyos8u9RQKAWRNchFh0p0FCS9xNrn3e+yHnt+p6mOOF2gMzfXT/M2hq -KQNmc5D3yNoH2smWoz7F3XsRjIB1Ie4VWoRRaGEy7RwcwiDfcaemD0rQug6iqH7N -oomF6f3R4DyvVVLUkQIDAQABMA0GCSqGSIb3DQEBCwUAA4IBAQB/8SX6qyKsOyex -v3wubgN3FPyU9PqMfEzrFM6X5sax0VMVbSnekZrrXpdnXYV+3FBu2GLLQc900ojj -vKD+409JIriTcwdFGdLrQPTCRWkEOae8TlXpTxuNqJfCPVNxFN0znoat1bSRsX1U -K0mfEETQ3ARwlTkrF9CM+jkU3k/pnc9MoCLif8P7OAF38AmIbuTUG6Gpzy8RytJn -m5AiA3sds5R0rpGUu8mFeBpT6jIA1QF2g+QNHKOQcfJdCdfqTjKw5y34hjFqbWG9 -RxWGeGNZkhC/jADCt+m+R6+hlyboLuIcVp8NJw6CGbr1+k136z/Dj+Fdhm6FzF7B -qULeRQJ+ ------END CERTIFICATE----- ------BEGIN PRIVATE KEY----- -MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCiMa59BpNVP0Ml -Yn/OiAZ+uvIQldhXL1zjjU7kysuVHHucIhYhSzY0SEcMl2omNg5N2aQr3xWF0ykM -9nDGOC+9muHjeVK0FEhbfNnG/vjutKXZW2lKoP/pQ1Yl7m7hJHuiBrgl8hM4oeD/ -4BLGc7Gn+K7qhPqMEvpPspI+HpF/b6K3auyOGS1Gt4UMnaHKug2rXDhlU0Jw0+BQ -jKizy71FAoBZE1yEWHSnQUJL3E2ufd77Iee36nqY44XaAzN9dP8zaGopA2ZzkPfI -2gfayZajPsXdexGMgHUh7hVahFFoYTLtHBzCIN9xp6YPStC6DqKofs2iiYXp/dHg -PK9VUtSRAgMBAAECggEANjn0A3rqUUr4UQxwfIV/3mj0O1VN4kBEhxOcd+PRUsYW -EapXycPSmII9ttj8tU/HUoHcYIqSMI7bn6jZJXxtga/BrALJAsnxMx031k8yvOQK -uvPT7Q6M4NkReVcRHRbMeuxSLuWTRZDhn8qznEPb9rOvD1tsRN6nb3PdbwVbUcZh -2F6JDrTyI/Df6nrYQAWOEe2ay7tzgrNYE4vh+DW7oVmyHRgFYA+DIG5Q+7OVWeW5 -bwYYPKlo4/B0L+GfMKfMVZ+5TvFWAK0YD1e/CW1Gv+i/8dWm4O7UNGg5mTnrIcy1 -g5wkKbyea02/np2B/XBsSWXDl6rTDHL7ay0rH2hjEQKBgQDMKSm3miQTIcL/F2kG -ieapmRtSc7cedP967IwUfjz4+pxPa4LiU47OCGp1bmUTuJAItyQyu/5O3uLpAriD -PTU+oVlhqt+lI6+SJ4SIYw01/iWI3EF2STwXVnohWG1EgzuFM/EqoB+mrodNONfG -UmP58vI9Is8fdugXgpTz4Yq9pQKBgQDLYJoyMVrYTvUn5oWft8ptsWZn6JZXt5Bd -aXh+YhNmtCrSORL3XjcH4yjlcn7X8Op33WQTbPo7QAJ1CumJzAI88BZ/8za638xb -nLueviZApCt0bNMEEdxDffxHFc5TyHE+obMKFfApbCnD0ggO6lrZ8jK9prArLOCp -mRU9SSRffQKBgAjoBszeqZI4F9SfBdLmMyzU5A89wxBOFFMdfKLsOua1sBn627PZ -51Hvpg1HaptoosfujWK1NsvkB0wY9UmsYuU/jrGnDaibnO4oUSzN/WaMlsCYszZg -zYFLIXrQ67tgajlOYcf1Qkw4MujYgPlC4N+njI/EM/rwagGUjcDx5uaNAoGASyqz -EuYG63eTSGH89SEaohw0+yaNmnHv23aF4EAjZ4wjX3tUtTSPJk0g6ly84Nbb8d1T -hZJ7kbaAsf2Mfy91jEw4JKYhjkP05c8x0OP6g12p6efmvdRUEmXX/fXjQjgNEtb0 -sz+UedrOPN+9trWLSo4njsyyw+JcTpKTtQj5dokCgYEAg9Y3msg+GvR5t/rPVlKd -keZkrAp0xBJZgqG7CTPXWS1FjwbAPo7x4ZOwtsgjCuE52lar4j+r2Il+CDYeLfxN -h/Jfn6S9ThUh+B1PMvKMMnJUahg8cVL8uQuBcbAy8HPRK78WO2BTnje44wFAJwTc -0liuYqVxZIRlFLRl8nGqog8= ------END PRIVATE KEY----- -""", #7 -"""-----BEGIN CERTIFICATE----- -MIICojCCAYoCAQEwDQYJKoZIhvcNAQELBQAwFzEVMBMGA1UEAwwMbmV3cGJfdGhp -bmd5MB4XDTIwMDEwMjAxNTExMloXDTIxMDEwMTAxNTExMlowFzEVMBMGA1UEAwwM -bmV3cGJfdGhpbmd5MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAu9oO -cFlNukUcLfFrfkEaUiilcHLmn5OokQbj95CGd2ehQCCVwrkunYLBisthRaancFFb -/yM998B0IUsKTsoLi5DAN3/SkSm6GiQIGO05E4eBPljwJ61QQMxh8+1TwQ9HTun1 -ZE1lhVN1aRmI9VsbyTQLjXh9OFNLSJEKb29mXsgzYwYwNOvo+idzXpy4bMyNoGxY -Y+s2FIKehNHHCv4ravDn8rf6DtDOvyN4d0/QyNws9FpAZMXmLwtBJ9exOqKFW43w -97NxgdNiTFyttrTKTi0b+9v3GVdcEZw5b2RMIKi6ZzPof6/0OlThK6C3xzFK3Bp4 -PMjTfXw5yyRGVBnZZwIDAQABMA0GCSqGSIb3DQEBCwUAA4IBAQA4Ms6LqzMu757z -bxISiErRls6fcnq0fpSmiPNHNKM7YwG9KHYwPT6A0UMt30zDwNOXCQBI19caGeeO -MLPWa7Gcqm2XZB2jQwvLRPeFSy9fm6RzJFeyhrh/uFEwUetwYmi/cqeIFDRDBQKn -bOaXkBk0AaSmI5nRYfuqpMMjaKOFIFcoADw4l9wWhv6DmnrqANzIdsvoSXi5m8RL -FcZQDZyHFlHh3P3tLkmQ7ErM2/JDwWWPEEJMlDm/q47FTOQSXZksTI3WRqbbKVv3 -iQlJjpgi9yAuxZwoM3M4975iWH4LCZVMCSqmKCBt1h9wv4LxqX/3kfZhRdy1gG+j -41NOSwJ/ ------END CERTIFICATE----- ------BEGIN PRIVATE KEY----- -MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQC72g5wWU26RRwt -8Wt+QRpSKKVwcuafk6iRBuP3kIZ3Z6FAIJXCuS6dgsGKy2FFpqdwUVv/Iz33wHQh -SwpOyguLkMA3f9KRKboaJAgY7TkTh4E+WPAnrVBAzGHz7VPBD0dO6fVkTWWFU3Vp -GYj1WxvJNAuNeH04U0tIkQpvb2ZeyDNjBjA06+j6J3NenLhszI2gbFhj6zYUgp6E -0ccK/itq8Ofyt/oO0M6/I3h3T9DI3Cz0WkBkxeYvC0En17E6ooVbjfD3s3GB02JM -XK22tMpOLRv72/cZV1wRnDlvZEwgqLpnM+h/r/Q6VOEroLfHMUrcGng8yNN9fDnL -JEZUGdlnAgMBAAECggEALlZdlW0R9U6y4spYf65Dddy84n4VUWu0+wE+HoUyBiYz -6oOfLYdMbmIgp8H/XpT7XINVNBxXXtPEUaoXAtRoAKdWItqO8Gvgki4tKSjrGVwl -j2GU69SepT1FNExoiojgSCEB/RnyXu71WVWJKSyuL/V8nAsKqGgze9T7Q/2wvNQt -SQqLxZlrWF0P8WqaAiSrHV4GnDrdeF+k1KBo2+pSaDNv6cNwOyVG8EII9tqhF8kj -6nD6846ish6OqmlSisaSGopJZL1DCQzszFMxKd2+iBDY7Kn6hVIhRaNnaZUFhpKM -dNh6hBqOycMepAp0sz5pdo+fxpifkoR/cPWgyC3QkQKBgQDixe9VsiZ7u2joxF/9 -JcAExKhqE28OUmIwt6/j+uzYShxN6Oo9FUo3ICtAPCCFsjhvb3Qum7FspmxrqtNy -fzclibZJPO8ey2PzqaiOfiVfgJmNSvoCOdgM4OqFLtRO6eSTzhJeI4VPrPcq/5la -0FuOi1WZs/Au9llqLqGSDH3UAwKBgQDUD/bSJbOk5SvNjFtFm0ClVJr66mJ5e4uN -4VGv8KGFAJ+ExIxujAukfKdwLjS1wEy2RePcshfT8Y9FVh/Q1KzzrQi3Gwmfq1G6 -Dpu2HlJpaZl+9T81x2KS8GP3QNczWMe2nh7Lj+6st+b4F+6FYbVTFnHaae27sXrD -XPX15+uxzQKBgGy+pBWBF4kwBo/QU4NuTdU7hNNRPGkuwl1ASH1Xv6m8aDRII8Nk -6TDkITltW98g5oUxehI7oOpMKCO9SCZYsNY0YpBeQwCOYgDfc6/Y+A0C+x9RO/BD -UsJiPLPfD/pDmNPz9sTj3bKma+RXq29sCOujD0pkiiHLCnerotkJWnGHAoGAAkCJ -JoIv/jhQ1sX+0iZr8VWMr819bjzZppAWBgBQNtFi4E4WD7Z9CSopvQ9AkA2SwvzL -BrT9e8q88sePXvBjRdM4nHk1CPUQ0SEGllCMH4J3ltmT6kZLzbOv3BhcMLdop4/W -U+MbbcomMcxPRCtdeZxraR5m3+9qlliOZCYqYqECgYA5eLdxgyHxCS33QGFHRvXI -TLAHIrr7wK1xwgkmZlLzYSQ8Oqh1UEbgoMt4ulRczP2g7TCfvANw2Sw0H2Q5a6Fj -cnwVcXJ38DLg0GCPMwzE8dK7d8tKtV6kGiKy+KFvoKChPjE6uxhKKmCJaSwtQEPS -vsjX3iiIgUQPsSz8RrNFfQ== ------END PRIVATE KEY----- -""", #8 -"""-----BEGIN CERTIFICATE----- -MIICojCCAYoCAQEwDQYJKoZIhvcNAQELBQAwFzEVMBMGA1UEAwwMbmV3cGJfdGhp -bmd5MB4XDTIwMDEwMjAxNTExMloXDTIxMDEwMTAxNTExMlowFzEVMBMGA1UEAwwM -bmV3cGJfdGhpbmd5MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA5DNu -CKhhl6wCbgoCkFemwJh3ATbAjhInHpvQWIFDfSK1USElCKxqosIxiBQCx3Zs2d/U -GeIA7QAM2atNdXaateacEaKMmGE9LEtO0Dg5lmT43WzmGkG9NmCwK3JjAekc5S9d -HKNtEQo7o8RKfj81zlDSq2kzliy98cimk24VBBGkS2Cn7Vy/mxMCqWjQazTXbpoS -lXw6LiY5wFXQmXOB5GTSHvqyCtBQbOSSbJB77z/fm7bufTDObufTbJIq53WPt00Y -f+JNnzkX1X0MaBCUztoZwoMaExWucMe/7xsQ46hDn6KB4b0lZk+gsK45QHxvPE1R -72+ZkkIrGS/ljIKahQIDAQABMA0GCSqGSIb3DQEBCwUAA4IBAQDib1653CneSmy2 -gYzGeMlrI05Jqo3JuHNMQHzAjIrb4ee57VA4PTQa1ygGol/hVv6eTvZr3p2ospDS -5Kfwj1HLO4jSMX1Bnm1FG0naQogz2CD3xfYjbYOVRhAxpld1MNyRveIOhDRARY7N -XNAaNPZ1ALrwbENSYArr18xDzgGWe/dgyRCEpCFIsztiA+7jGvrmAZgceIE8K3h3 -fkvNmXBH58ZHAGTiyRriBZqS+DXrBrQOztXSJwFnOZnRt6/efeBupt8j5hxVpBLW -vtjpBc23uUcbbHOY2AW2Bf+vIr4/LmJ/MheKV+maa2990vmC93tvWlFfc74mgUkW -HJfXDmR6 ------END CERTIFICATE----- ------BEGIN PRIVATE KEY----- -MIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQDkM24IqGGXrAJu -CgKQV6bAmHcBNsCOEicem9BYgUN9IrVRISUIrGqiwjGIFALHdmzZ39QZ4gDtAAzZ -q011dpq15pwRooyYYT0sS07QODmWZPjdbOYaQb02YLArcmMB6RzlL10co20RCjuj -xEp+PzXOUNKraTOWLL3xyKaTbhUEEaRLYKftXL+bEwKpaNBrNNdumhKVfDouJjnA -VdCZc4HkZNIe+rIK0FBs5JJskHvvP9+btu59MM5u59NskirndY+3TRh/4k2fORfV -fQxoEJTO2hnCgxoTFa5wx7/vGxDjqEOfooHhvSVmT6CwrjlAfG88TVHvb5mSQisZ -L+WMgpqFAgMBAAECggEABTdPuo7uvCLIY2+DI319aEWT4sk3mYe8sSxqlLtPqZqT -fmk9iXc3cMTzkOK0NY71af19waGy17f6kzchLCAr5SCCTLzkbc87MLn/8S530oI4 -VgdZMxxxkL6hCD0zGiYT7QEqJa9unMcZGeMwuLYFKtQaHKTo8vPO26n0dMY9YLxj -cNUxsKLcKk8dbfKKt4B4fZgB7nU0BG9YbKYZ3iZ7/3mG+6jA6u+VYc/WHYQjTmpL -oLFN7NOe3R7jIx/kJ1OqNWqsFoLpyiiWd1Mr0l3EdD1kCudptMgD8hd++nx2Yk2w -K4+CpOVIN/eCxDDaAOJgYjCtOayVwUkDAxRRt9VnAQKBgQD5s1j6RJtBNTlChVxS -W3WpcG4q8933AiUY/Chx0YTyopOiTi7AGUaA8AOLFBcO2npa+vzC+lvuOyrgOtVW -sD10H2v5jNKlbeBp+Q9rux2LAyp4TvzdXWKhVyZrdtITF0hn6vEYNp7MtyWRFb1O -3Ie5HQBPHtzllFOMynacjOdjpQKBgQDp9TrbfOmwGWmwPKmaFKuy8BKxjJM+ct0X -4Xs1uSy9Z9Y8QlDNbNaooI8DA1NY0jDVHwemiGC4bYsBNKNRcbI0s2nr0hQMft42 -P/NpugHv0YXiVz+5bfim4woTiHHbfREqchlIGo3ryClAiDU9fYZwTOtb9jPIhX3G -9v+OsoMlYQKBgQDJUQW90S5zJlwh+69xXvfAQjswOimNCpeqSzK4gTn0/YqV4v7i -Nf6X2eqhaPMmMJNRYuYCtSMFMYLiAc0a9UC2rNa6/gSfB7VU+06phtTMzSKimNxa -BP6OIduB7Ox2I+Fmlw8GfJMPbeHF1YcpW7e5UV58a9+g4TNzYZC7qwarWQKBgQCA -FFaCbmHonCD18F/REFvm+/Lf7Ft3pp5PQouXH6bUkhIArzVZIKpramqgdaOdToSZ -SAGCM8rvbFja8hwurBWpMEdeaIW9SX8RJ/Vz/fateYDYJnemZgPoKQcNJnded5t8 -Jzab+J2VZODgiTDMVvnQZOu8To6OyjXPRM0nK6cMQQKBgQDyX44PHRRhEXDgJFLU -qp2ODL54Qadc/thp2m+JmAvqmCCLwuYlGpRKVkLLuZW9W6RlVqarOC3VD3wX5PRZ -IsyCGLi+Jbrv9JIrYUXE80xNeQVNhrrf02OW0KHbqGxRaNOmp1THPw98VUGR2J/q -YAp6XUXU7LEBUrowye+Ty2o7Lg== ------END PRIVATE KEY----- -""", #9 -"""-----BEGIN CERTIFICATE----- -MIICojCCAYoCAQEwDQYJKoZIhvcNAQELBQAwFzEVMBMGA1UEAwwMbmV3cGJfdGhp -bmd5MB4XDTIwMDEwMjAxNTExMVoXDTIxMDEwMTAxNTExMVowFzEVMBMGA1UEAwwM -bmV3cGJfdGhpbmd5MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA1k2R -PWYihftppo3CoxeseFwgg7guxZVkP7aAur5uBzSeAB7sBG1G2bRrwMX71S4xPwot -zYiEoxUrTStUqEKjL2aozfHsXnHZ7kwwUgZFDZUg+ve2tZDA3HCUr4tLYKlyFqpx -2nCouc45MjQ4wAxRl4rQxIUG2uSTzvP+xXtjoJYMIEEyCpcsRXfqfVkEUe9nrPsF -0Ibzk7Cyt75HDI4uEzBuHux0DYuGy6R02jz/vf/dIZ4WepjSY06xpblTHZgieDRX -fU2+YOcvb0eDHyA8Q5p8ropK71MNIP5+kffFd90SVr4EkCA8S+cd6FdKQasRr+jF -9MUhMS4ObvlrYTG+hwIDAQABMA0GCSqGSIb3DQEBCwUAA4IBAQCy62MZ3+59/VpX -c9Hsmb4/BMWt0irLJit4w4SkuYKGFLCMKZI4LN4pEkXaiE1eqF2DNS1qOvl5luty -Zz4oggrqilwuFeH98o9Zeg9SYnouuypORVP/3DPbJF/jiQg5J8kJb1sy+BjRiT8I -5X6/cCBYT+MljFz5tpqWOtWTgA30e1BV8JFj8F4dgUcWsAVT/I4l9zgMLUnhcO6E -wLtEE0I6aT1RHJB28ndwJzj4La98Oirw7LAEAWbExWYB90ypLaGY+JVJe3f5fijC -fJpQ2mbs4syXDmb5bU2C2pGPTKZPcyx15iQrq1uHInD0facOw+pmllAFxuG96lA1 -+o2VzKwP ------END CERTIFICATE----- ------BEGIN PRIVATE KEY----- -MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDWTZE9ZiKF+2mm -jcKjF6x4XCCDuC7FlWQ/toC6vm4HNJ4AHuwEbUbZtGvAxfvVLjE/Ci3NiISjFStN -K1SoQqMvZqjN8execdnuTDBSBkUNlSD697a1kMDccJSvi0tgqXIWqnHacKi5zjky -NDjADFGXitDEhQba5JPO8/7Fe2OglgwgQTIKlyxFd+p9WQRR72es+wXQhvOTsLK3 -vkcMji4TMG4e7HQNi4bLpHTaPP+9/90hnhZ6mNJjTrGluVMdmCJ4NFd9Tb5g5y9v -R4MfIDxDmnyuikrvUw0g/n6R98V33RJWvgSQIDxL5x3oV0pBqxGv6MX0xSExLg5u -+WthMb6HAgMBAAECggEAeCyRSNwQeg/NZD/UqP6qkegft52+ZMBssinWsGH/c3z3 -KVwtwCHDfGvnjPe5TAeWSCKeIsbukkFZwfGNjLmppvgrqymCAkhYDICfDDBF4uMA -1pu40sJ01Gkxh+tV/sOmnb1BEVzh0Sgq/NM6C8ActR18CugKOw+5L3G2KeoSqUbT -2hcPUsnik10KwqW737GQW4LtEQEr/iRmQkxI3+HBzvPWjFZzjOcpUph+FW5TXtaU -T26mt1j+FjbdvvhCuRMY/VZBJ5h1RKU95r57F1AjW/C0RRJ8FxR1CeSy4IlmQBrh -6wAa3Tdm0k/n4ZspC9bF5eVTJEtb0AohiYZrIa8MuQKBgQD8yjCLYa41H304odCx -NwPRJcmlIk5YGxPrhHAT9GEgU6n/no7YMVx1L7fNLcMjAyx54jauEU7J19Aki7eV -SIdU9TwqmkOAFfM6TOEJZiOi66gABOxeK2yDyfmR6Apaw3caku4O058t4KVwHSCB -DanYCMzxCBqS9jUTTyAh0fMg6wKBgQDZBkIukg3FKPor5LzkUXIKnNHYPfHbERHw -piWS6GZwqhuWNlOCWxiBR4rEUU/RbFQZw/FCi5OuAk2lBC0LBmC0/Sz4/+xDdCbv -uNhMOTRcy9nFVpmpIWCx4N/KmXHEuFxli/JNXux7iki74AVC9VPrAt/kCvwf06Df -oDb8ljdR1QKBgQChVOD6c5Lc8IXYeN1Z3IShHH6+11AsxstFyjZFZff+y6Z5L1Z2 -/7nESHoDhqs9Uy81cnv3R7CC/Ssnx8uYiLtmK0UE44Mk4d1jXeFZQEiKF+AWcw3v -Y8NTsLmItxC0sH75BMDN0Z2LiA3Nqaku8+trpuI1Cjj7hgqFkkAtlXKXlQKBgBMb -c/Q5s7CqHOyEZQUNDqdUiz0opwSMijHPzvsSLwK4V1lwSwXtE0k+jT8fkZF0oirq -j3E2bLgjR8bBiV2xIA6PQ8hgb+K4dT0h3xlG6A9Le07egwTbBXJjxBBIVjXlrWzb -V2fsdZGi6ShxXsU4aD0GscOYG/6JWV6W8oBmkVRJAoGAepIZ+OYmFjb7uxdh4EtP -hluEtx5bLOLuo6c0S149omUXUhbsuyzTZS6Ip9ySDMnK3954c4Q4WJ4yQKixQNVq -78aDfy4hP/8TE/Q9CRddUof2P33PJMhVNqzTRYMpqV+zxifvtw3hoDTLKHTQxCR2 -M1+O4VvokU5pBqUpGXiMDfs= ------END PRIVATE KEY----- -""", #10 -"""-----BEGIN CERTIFICATE----- -MIICojCCAYoCAQEwDQYJKoZIhvcNAQELBQAwFzEVMBMGA1UEAwwMbmV3cGJfdGhp -bmd5MB4XDTIwMDEwMjAxNTExMVoXDTIxMDEwMTAxNTExMVowFzEVMBMGA1UEAwwM -bmV3cGJfdGhpbmd5MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAnbCU -M37hG7zrCyyJEI6pZmOomnI+CozbP5KAhWSV5y7R5H6lcAEG2UDV+lCUxHT2ufOa -i1H16bXyBt7VoMTHIH50S58NUCUEXcuRWVR16tr8CzcTHQAkfIrmhY2XffPilX7h -aw35UkoVmXcqSDNNJD6jmvWexvmbhzVWW8Vt5Pivet2/leVuqPXB54/alSbkC74m -x6X5XKQc6eyPsb1xvNBuiSpFzdqbEn7lUwj6jFTkh9tlixgmgx+J0XoQXbawyrAg -rcIQcse/Ww+KBA1KSccFze+XBTbIull4boYhbJqkb6DW5bY7/me2nNxE9DRGwq+S -kBsKq3YKeCf8LEhfqQIDAQABMA0GCSqGSIb3DQEBCwUAA4IBAQAD+tWGFhINYsWT -ibKWlCGgBc5uB7611cLCevx1yAL6SaOECVCQXzaaXIaETSbyY03UO2yBy3Pl10FV -GYXLrAWTFZsNVJm55XIibTNw1UBPNwdIoCSzAYuOgMF0GHhTTQU0hNYWstOnnE2T -6lSAZQZFkaW4ZKs6sUp42Em9Bu99PehyIgnw14qb9NPg5qKdi2GAvkImZCrGpMdK -OF31U7Ob0XQ0lxykcNgG4LlUACd+QxLfNpmLBZUGfikexYa1VqBFm3oAvTt8ybNQ -qr7AKXDFnW75aCBaMpQWzrstA7yYZ3D9XCd5ZNf6d08lGM/oerDAIGnZOZPJgs5U -FaWPHdS9 ------END CERTIFICATE----- ------BEGIN PRIVATE KEY----- -MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCdsJQzfuEbvOsL -LIkQjqlmY6iacj4KjNs/koCFZJXnLtHkfqVwAQbZQNX6UJTEdPa585qLUfXptfIG -3tWgxMcgfnRLnw1QJQRdy5FZVHXq2vwLNxMdACR8iuaFjZd98+KVfuFrDflSShWZ -dypIM00kPqOa9Z7G+ZuHNVZbxW3k+K963b+V5W6o9cHnj9qVJuQLvibHpflcpBzp -7I+xvXG80G6JKkXN2psSfuVTCPqMVOSH22WLGCaDH4nRehBdtrDKsCCtwhByx79b -D4oEDUpJxwXN75cFNsi6WXhuhiFsmqRvoNbltjv+Z7ac3ET0NEbCr5KQGwqrdgp4 -J/wsSF+pAgMBAAECggEAPSu1ofBTRN5ZU4FYPlsJLdX1Hsy4coFHv/aF8rkdSYwp -EflrFfLgBEEZgLvnqfoxh9sPFYKa4amaFL42ouIS2PEVDgzKLk/dzMDeRof0IkIG -yhb4TCS1ArcjS6WsociNGi8ZJN1L3Xctv9WxSkbUYv4Fm2Qyzr8fbSjssjb5NXwD -K11fsj6Pfy/mQrI0TSTlzWC7ARIlCMTWQ8G8zEU6bMFIG6DMjt2J4VgYVXUKetZA -VPuS+pwkH2obQe6FLRuiNxH4GitVAASYPea6foER4AggBMRp8q8F6+WssjoyEORb -0sJxmnxhznoTRMCuTsUj6XMgmOTOnA3lQXsIB0DYcQKBgQDO6mMRVVFWzgkE9Q5/ -36n06KvGYF9TCRDL9vRC8kCqcGd1Hy6jRj0D8049KUHaN74pfWg6gsQjPkKzwKnC -vxNl72tVvLqm7Fo531BGfKK/46ZvxeWMMraNW4+9LhwMPu2LN5OEdwwCgyaURpxh -ktCp+RrGjz08Kn82X1jJPdwxDQKBgQDDGMvZ7ZUDGq5+RJkmHJ58lQtiaMZclmYV -R9YwOxJV6ino3EYrGOtUkqiemgAACdMWE/JMJlB1/JINawJwUsZ2XDp/9jNLPgLc -gphCmagaO34U/YMaJbJIK2gkCX7p8EcD+x45qWa0bEMPW38QfN/qQdUPjNmpuIiI -Zleyl1TqDQKBgQCvIoat0ighsAzETGN0aqzhJdrW8xVcJA06hpFi5MdFPBTldno0 -KqxUXqj3badWe94SIhqJg8teBUHSAZ3uv2o82nRgQnk99km8OD8rGi1q+9YRP1C2 -5OnNJhW4y4FkABNxxZ2v/k+FBNsvn8CXefvyEm3OaMks1s+MBxIQa7KnNQKBgFwX -HUo+GiN/+bPCf6P8yFa4J8qI+HEF0SPkZ9cWWx5QzP2M1FZNie++1nce7DcYbBo0 -yh9lyn8W/H328AzDFckS2c5DEY1HtSQPRP3S+AWB5Y7U54h1GMV2L88q6ExWzb60 -T10aeE9b9v+NydmniC5UatTPQIMbht8Tp/u18TAVAoGBAJphAfiqWWV2M5aBCSXq -WxLZ71AJ0PZBmRa/9iwtccwXQpMcW6wHK3YSQxci+sB97TElRa3/onlVSpohrUtg -VCvCwfSHX1LmrfWNSkoJZwCQt+YYuMqW86K0tzLzI1EMjIH9LgQvB6RR26PZQs+E -jr1ZvRc+wPTq6sxCF1h9ZAfN ------END PRIVATE KEY----- -""", #11 -] - -# To disable the pre-computed tub certs, uncomment this line. -# SYSTEM_TEST_CERTS = [] - -def flush_but_dont_ignore(res): - d = flushEventualQueue() - def _done(ignored): - return res - d.addCallback(_done) - return d - -def _render_config(config): - """ - Convert a ``dict`` of ``dict`` of ``unicode`` to an ini-format string. - """ - return u"\n\n".join(list( - _render_config_section(k, v) - for (k, v) - in config.items() - )) - -def _render_config_section(heading, values): - """ - Convert a ``unicode`` heading and a ``dict`` of ``unicode`` to an ini-format - section as ``unicode``. - """ - return u"[{}]\n{}\n".format( - heading, _render_section_values(values) - ) - -def _render_section_values(values): - """ - Convert a ``dict`` of ``unicode`` to the body of an ini-format section as - ``unicode``. - """ - return u"\n".join(list( - u"{} = {}".format(k, v) - for (k, v) - in sorted(values.items()) - )) - - -class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin): - - def setUp(self): - self.port_assigner = SameProcessStreamEndpointAssigner() - self.port_assigner.setUp() - self.addCleanup(self.port_assigner.tearDown) - - self.sparent = service.MultiService() - self.sparent.startService() - - def tearDown(self): - log.msg("shutting down SystemTest services") - d = self.sparent.stopService() - d.addBoth(flush_but_dont_ignore) - return d - - def getdir(self, subdir): - return os.path.join(self.basedir, subdir) - - def add_service(self, s): - s.setServiceParent(self.sparent) - return s - - def _create_introducer(self): - """ - :returns: (via Deferred) an Introducer instance - """ - iv_dir = self.getdir("introducer") - if not os.path.isdir(iv_dir): - _, port_endpoint = self.port_assigner.assign(reactor) - introducer_config = ( - u"[node]\n" - u"nickname = introducer \N{BLACK SMILING FACE}\n" + - u"web.port = {}\n".format(port_endpoint) - ).encode("utf-8") - - fileutil.make_dirs(iv_dir) - fileutil.write( - os.path.join(iv_dir, 'tahoe.cfg'), - introducer_config, - ) - if SYSTEM_TEST_CERTS: - os.mkdir(os.path.join(iv_dir, "private")) - f = open(os.path.join(iv_dir, "private", "node.pem"), "w") - f.write(SYSTEM_TEST_CERTS[0]) - f.close() - return create_introducer(basedir=iv_dir) - - def _get_introducer_web(self): - with open(os.path.join(self.getdir("introducer"), "node.url"), "r") as f: - return f.read().strip() - - @inlineCallbacks - def set_up_nodes(self, NUMCLIENTS=5): - """ - Create an introducer and ``NUMCLIENTS`` client nodes pointed at it. All - of the nodes are running in this process. - - As a side-effect, set: - - * ``numclients`` to ``NUMCLIENTS`` - * ``introducer`` to the ``_IntroducerNode`` instance - * ``introweb_url`` to the introducer's HTTP API endpoint. - - :param int NUMCLIENTS: The number of client nodes to create. - - :return: A ``Deferred`` that fires when the nodes have connected to - each other. - """ - self.numclients = NUMCLIENTS - - self.introducer = yield self._create_introducer() - self.add_service(self.introducer) - self.introweb_url = self._get_introducer_web() - yield self._set_up_client_nodes() - - @inlineCallbacks - def _set_up_client_nodes(self): - q = self.introducer - self.introducer_furl = q.introducer_url - self.clients = [] - basedirs = [] - for i in range(self.numclients): - basedirs.append((yield self._set_up_client_node(i))) - - # start clients[0], wait for it's tub to be ready (at which point it - # will have registered the helper furl). - c = yield client.create_client(basedirs[0]) - c.setServiceParent(self.sparent) - self.clients.append(c) - c.set_default_mutable_keysize(TEST_RSA_KEY_SIZE) - - with open(os.path.join(basedirs[0],"private","helper.furl"), "r") as f: - helper_furl = f.read() - - self.helper_furl = helper_furl - if self.numclients >= 4: - with open(os.path.join(basedirs[3], 'tahoe.cfg'), 'a+') as f: - f.write( - "[client]\n" - "helper.furl = {}\n".format(helper_furl) - ) - - # this starts the rest of the clients - for i in range(1, self.numclients): - c = yield client.create_client(basedirs[i]) - c.setServiceParent(self.sparent) - self.clients.append(c) - c.set_default_mutable_keysize(TEST_RSA_KEY_SIZE) - log.msg("STARTING") - yield self.wait_for_connections() - log.msg("CONNECTED") - # now find out where the web port was - self.webish_url = self.clients[0].getServiceNamed("webish").getURL() - if self.numclients >=4: - # and the helper-using webport - self.helper_webish_url = self.clients[3].getServiceNamed("webish").getURL() - - def _generate_config(self, which, basedir): - config = {} - - except1 = set(range(self.numclients)) - {1} - feature_matrix = { - ("client", "nickname"): except1, - - # client 1 has to auto-assign an address. - ("node", "tub.port"): except1, - ("node", "tub.location"): except1, - - # client 0 runs a webserver and a helper - # client 3 runs a webserver but no helper - ("node", "web.port"): {0, 3}, - ("node", "timeout.keepalive"): {0}, - ("node", "timeout.disconnect"): {3}, - - ("helper", "enabled"): {0}, - } - - def setconf(config, which, section, feature, value): - if which in feature_matrix.get((section, feature), {which}): - config.setdefault(section, {})[feature] = value - - setnode = partial(setconf, config, which, "node") - sethelper = partial(setconf, config, which, "helper") - - setnode("nickname", u"client %d \N{BLACK SMILING FACE}" % (which,)) - - tub_location_hint, tub_port_endpoint = self.port_assigner.assign(reactor) - setnode("tub.port", tub_port_endpoint) - setnode("tub.location", tub_location_hint) - - _, web_port_endpoint = self.port_assigner.assign(reactor) - setnode("web.port", web_port_endpoint) - setnode("timeout.keepalive", "600") - setnode("timeout.disconnect", "1800") - - sethelper("enabled", "True") - - iyaml = ("introducers:\n" - " petname2:\n" - " furl: %s\n") % self.introducer_furl - iyaml_fn = os.path.join(basedir, "private", "introducers.yaml") - fileutil.write(iyaml_fn, iyaml) - - return _render_config(config) - - def _set_up_client_node(self, which): - basedir = self.getdir("client%d" % (which,)) - fileutil.make_dirs(os.path.join(basedir, "private")) - if len(SYSTEM_TEST_CERTS) > (which + 1): - f = open(os.path.join(basedir, "private", "node.pem"), "w") - f.write(SYSTEM_TEST_CERTS[which + 1]) - f.close() - config = self._generate_config(which, basedir) - fileutil.write(os.path.join(basedir, 'tahoe.cfg'), config) - return basedir - - def bounce_client(self, num): - c = self.clients[num] - d = c.disownServiceParent() - # I think windows requires a moment to let the connection really stop - # and the port number made available for re-use. TODO: examine the - # behavior, see if this is really the problem, see if we can do - # better than blindly waiting for a second. - d.addCallback(self.stall, 1.0) - - @defer.inlineCallbacks - def _stopped(res): - new_c = yield client.create_client(self.getdir("client%d" % num)) - self.clients[num] = new_c - new_c.set_default_mutable_keysize(TEST_RSA_KEY_SIZE) - new_c.setServiceParent(self.sparent) - d.addCallback(_stopped) - d.addCallback(lambda res: self.wait_for_connections()) - def _maybe_get_webport(res): - if num == 0: - # now find out where the web port was - self.webish_url = self.clients[0].getServiceNamed("webish").getURL() - d.addCallback(_maybe_get_webport) - return d - - @defer.inlineCallbacks - def add_extra_node(self, client_num, helper_furl=None, - add_to_sparent=False): - # usually this node is *not* parented to our self.sparent, so we can - # shut it down separately from the rest, to exercise the - # connection-lost code - basedir = FilePath(self.getdir("client%d" % client_num)) - basedir.makedirs() - config = "[client]\n" - if helper_furl: - config += "helper.furl = %s\n" % helper_furl - basedir.child("tahoe.cfg").setContent(config.encode("utf-8")) - private = basedir.child("private") - private.makedirs() - write_introducer( - basedir, - "default", - self.introducer_furl, - ) - - c = yield client.create_client(basedir.path) - self.clients.append(c) - c.set_default_mutable_keysize(TEST_RSA_KEY_SIZE) - self.numclients += 1 - if add_to_sparent: - c.setServiceParent(self.sparent) - else: - c.startService() - yield self.wait_for_connections() - defer.returnValue(c) - - def _check_connections(self): - for i, c in enumerate(self.clients): - if not c.connected_to_introducer(): - log.msg("%s not connected to introducer yet" % (i,)) - return False - sb = c.get_storage_broker() - connected_servers = sb.get_connected_servers() - connected_names = sorted(list( - connected.get_nickname() - for connected - in sb.get_known_servers() - if connected.is_connected() - )) - if len(connected_servers) != self.numclients: - wanted = sorted(list( - client.nickname - for client - in self.clients - )) - log.msg( - "client %s storage broker connected to %s, missing %s" % ( - i, - connected_names, - set(wanted) - set(connected_names), - ) - ) - return False - log.msg("client %s storage broker connected to %s, happy" % ( - i, connected_names, - )) - up = c.getServiceNamed("uploader") - if up._helper_furl and not up._helper: - log.msg("Helper fURL but no helper") - return False - return True - - def wait_for_connections(self, ignored=None): - return self.poll(self._check_connections, timeout=200) class CountingDataUploadable(upload.Data): bytes_read = 0 @@ -999,6 +115,7 @@ class CountingDataUploadable(upload.Data): self.interrupt_after_d.callback(self) return upload.Data.read(self, length) + class SystemTest(SystemTestMixin, RunBinTahoeMixin, unittest.TestCase): timeout = 180 From 5264108182ddd7a3a199e1dfb7af6db9dca08d18 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 13 Sep 2021 13:24:35 -0400 Subject: [PATCH 175/269] account for the archive message renumbering in the list migration --- NEWS.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NEWS.rst b/NEWS.rst index d70fac6b5..1cfc726ae 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -1188,7 +1188,7 @@ Precautions when Upgrading .. _`#1915`: https://tahoe-lafs.org/trac/tahoe-lafs/ticket/1915 .. _`#1926`: https://tahoe-lafs.org/trac/tahoe-lafs/ticket/1926 .. _`message to the tahoe-dev mailing list`: - https://lists.tahoe-lafs.org/pipermail/tahoe-dev/2013-March/008096.html + https://lists.tahoe-lafs.org/pipermail/tahoe-dev/2013-March/008079.html Release 1.9.2 (2012-07-03) From 87555ae3e434879393dd44de704483a18f7b9b5b Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 13 Sep 2021 15:51:57 -0400 Subject: [PATCH 176/269] Add missing __future__ imports. --- src/allmydata/test/common_system.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/allmydata/test/common_system.py b/src/allmydata/test/common_system.py index 6bda4e52d..9d14c8642 100644 --- a/src/allmydata/test/common_system.py +++ b/src/allmydata/test/common_system.py @@ -5,6 +5,11 @@ in ``allmydata.test.test_system``. Ported to Python 3. """ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + from future.utils import PY2 if PY2: # Don't import bytes since it causes issues on (so far unported) modules on Python 2. From bb626890ed1cf7e82a334fce9851dc7161cba36d Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 14 Sep 2021 08:57:32 -0400 Subject: [PATCH 177/269] Match review comment suggestions from previous PR. --- src/allmydata/test/test_istorageserver.py | 48 +++++++++++------------ 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index 7f2c9b7ff..78de0ae3d 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -268,14 +268,14 @@ class IStorageServerImmutableAPIsTestsMixin(object): Buckets that are not fully written are not returned by ``IStorageServer.get_buckets()`` implementations. """ - si = new_storage_index() + storage_index = new_storage_index() (_, allocated) = yield self.storage_server.allocate_buckets( - si, - new_secret(), - new_secret(), - set(range(5)), - 10, - Referenceable(), + storage_index, + renew_secret=new_secret(), + cancel_secret=new_secret(), + sharenums=set(range(5)), + allocated_size=10, + canary=Referenceable(), ) # Bucket 1 is fully written @@ -285,7 +285,7 @@ class IStorageServerImmutableAPIsTestsMixin(object): # Bucket 2 is partially written yield allocated[2].callRemote("write", 0, b"1" * 5) - buckets = yield self.storage_server.get_buckets(si) + buckets = yield self.storage_server.get_buckets(storage_index) self.assertEqual(set(buckets.keys()), {1}) @inlineCallbacks @@ -297,21 +297,21 @@ class IStorageServerImmutableAPIsTestsMixin(object): """ length = 256 * 17 - si = new_storage_index() + storage_index = new_storage_index() (_, allocated) = yield self.storage_server.allocate_buckets( - si, - new_secret(), - new_secret(), - set(range(1)), - length, - Referenceable(), + storage_index, + renew_secret=new_secret(), + cancel_secret=new_secret(), + sharenums=set(range(1)), + allocated_size=length, + canary=Referenceable(), ) total_data = _randbytes(256) * 17 yield allocated[0].callRemote("write", 0, total_data) yield allocated[0].callRemote("close") - buckets = yield self.storage_server.get_buckets(si) + buckets = yield self.storage_server.get_buckets(storage_index) bucket = buckets[0] for start, to_read in [ (0, 250), # fraction @@ -334,20 +334,20 @@ class IStorageServerImmutableAPIsTestsMixin(object): ``IStorageServer.get_buckets()`` does not result in error (other behavior is opaque at this level of abstraction). """ - si = new_storage_index() + storage_index = new_storage_index() (_, allocated) = yield self.storage_server.allocate_buckets( - si, - new_secret(), - new_secret(), - set(range(1)), - 10, - Referenceable(), + storage_index, + renew_secret=new_secret(), + cancel_secret=new_secret(), + sharenums=set(range(1)), + allocated_size=10, + canary=Referenceable(), ) yield allocated[0].callRemote("write", 0, b"0123456789") yield allocated[0].callRemote("close") - buckets = yield self.storage_server.get_buckets(si) + buckets = yield self.storage_server.get_buckets(storage_index) yield buckets[0].callRemote("advise_corrupt_share", b"OH NO") From 0aee960ea8a4a66d5b6174fdb8f2dccd4cb9954c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 14 Sep 2021 11:26:19 -0400 Subject: [PATCH 178/269] News file. --- newsfragments/3797.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3797.minor diff --git a/newsfragments/3797.minor b/newsfragments/3797.minor new file mode 100644 index 000000000..e69de29bb From d207c468553a5c76ff431fa725e7543cd1eae66c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 14 Sep 2021 11:26:23 -0400 Subject: [PATCH 179/269] First mutable test. --- src/allmydata/test/test_istorageserver.py | 62 +++++++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index 1c1383a2d..8de2be7a0 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -263,6 +263,62 @@ class IStorageServerImmutableAPIsTestsMixin(object): ) +class IStorageServerMutableAPIsTestsMixin(object): + """ + Tests for ``IStorageServer``'s mutable APIs. + + ``self.storage_server`` is expected to provide ``IStorageServer``. + + ``STARAW`` is short for ``slot_testv_and_readv_and_writev``. + """ + + # slot_testv_and_readv_and_writev + # TODO it's possible to write and then in separate call read + # TODO reads happen before (re)writes + # TODO write prevented if tests fail + # TODO reads beyond the edge + # TODO wrong write enabled prevents writes + # TODO write prevented if test data against empty share + # TODO writes can create additional shares if only some exist + # TODO later writes overwrite + + def new_secrets(self): + """Return a 3-tuple of secrets for STARAW calls.""" + return (new_secret(), new_secret(), new_secret()) + + def staraw(self, *args, **kwargs): + """Like ``slot_testv_and_readv_and_writev``, but less typing.""" + return self.storage_server.slot_testv_and_readv_and_writev(*args, **kwargs) + + @inlineCallbacks + def test_STARAW_reads(self): + """ + When data is written with + ``IStorageServer.slot_testv_and_readv_and_writev``, it can then be read + by a separate call using that API. + """ + secrets = self.new_secrets() + storage_index = new_storage_index() + (written, _) = yield self.staraw( + storage_index, + secrets, + tw_vectors={ + 0: ([], [(0, b"abcdefg")], 7), + 1: ([], [(0, b"0123456")], 7), + }, + r_vector=[], + ) + self.assertEqual(written, True) + + (_, reads) = yield self.staraw( + storage_index, + secrets, + tw_vectors={}, + r_vector=[(0, 7)], + ) + self.assertEqual(reads, {0: [b"abcdefg"], 1: [b"0123456"]}) + + class _FoolscapMixin(SystemTestMixin): """Run tests on Foolscap version of ``IStorageServer.""" @@ -293,3 +349,9 @@ class FoolscapImmutableAPIsTests( _FoolscapMixin, IStorageServerImmutableAPIsTestsMixin, AsyncTestCase ): """Foolscap-specific tests for immutable ``IStorageServer`` APIs.""" + + +class FoolscapMutableAPIsTests( + _FoolscapMixin, IStorageServerMutableAPIsTestsMixin, AsyncTestCase +): + """Foolscap-specific tests for immutable ``IStorageServer`` APIs.""" From 5b704ff12d9a22263750f02890eb4b42b53f7c37 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 14 Sep 2021 11:36:12 -0400 Subject: [PATCH 180/269] Another mutable test. --- src/allmydata/test/test_istorageserver.py | 46 +++++++++++++++++++++-- 1 file changed, 43 insertions(+), 3 deletions(-) diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index 8de2be7a0..780bb68c4 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -273,10 +273,10 @@ class IStorageServerMutableAPIsTestsMixin(object): """ # slot_testv_and_readv_and_writev - # TODO it's possible to write and then in separate call read - # TODO reads happen before (re)writes + # DONE it's possible to write and then in separate call read + # DONE reads happen before (re)writes # TODO write prevented if tests fail - # TODO reads beyond the edge + # TODO partial reads, reads beyond the edge # TODO wrong write enabled prevents writes # TODO write prevented if test data against empty share # TODO writes can create additional shares if only some exist @@ -318,6 +318,46 @@ class IStorageServerMutableAPIsTestsMixin(object): ) self.assertEqual(reads, {0: [b"abcdefg"], 1: [b"0123456"]}) + @inlineCallbacks + def test_SATRAW_reads_happen_before_writes_in_single_query(self): + """ + If a ``IStorageServer.slot_testv_and_readv_and_writev`` command + contains both reads and writes, the read returns results that precede + the write. + """ + secrets = self.new_secrets() + storage_index = new_storage_index() + (written, _) = yield self.staraw( + storage_index, + secrets, + tw_vectors={ + 0: ([], [(0, b"abcdefg")], 7), + }, + r_vector=[], + ) + self.assertEqual(written, True) + + # Read and write in same command; read happens before write: + (written, reads) = yield self.staraw( + storage_index, + secrets, + tw_vectors={ + 0: ([], [(0, b"X" * 7)], 7), + }, + r_vector=[(0, 7)], + ) + self.assertEqual(written, True) + self.assertEqual(reads, {0: [b"abcdefg"]}) + + # The write is available in next read: + (_, reads) = yield self.staraw( + storage_index, + secrets, + tw_vectors={}, + r_vector=[(0, 7)], + ) + self.assertEqual(reads, {0: [b"X" * 7]}) + class _FoolscapMixin(SystemTestMixin): """Run tests on Foolscap version of ``IStorageServer.""" From aa8001edf21dfb9a2a272f404998666894293b83 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 14 Sep 2021 12:30:45 -0400 Subject: [PATCH 181/269] Another test. --- src/allmydata/test/test_istorageserver.py | 58 ++++++++++++++++++++++- 1 file changed, 57 insertions(+), 1 deletion(-) diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index 780bb68c4..ddb855c4f 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -275,7 +275,8 @@ class IStorageServerMutableAPIsTestsMixin(object): # slot_testv_and_readv_and_writev # DONE it's possible to write and then in separate call read # DONE reads happen before (re)writes - # TODO write prevented if tests fail + # DONE write happens if test succeeds + # DONE write prevented if tests fail # TODO partial reads, reads beyond the edge # TODO wrong write enabled prevents writes # TODO write prevented if test data against empty share @@ -358,6 +359,61 @@ class IStorageServerMutableAPIsTestsMixin(object): ) self.assertEqual(reads, {0: [b"X" * 7]}) + @inlineCallbacks + def test_SATRAW_writen_happens_only_if_test_matches(self): + """ + If a ``IStorageServer.slot_testv_and_readv_and_writev`` includes both a + test and a write, the write succeeds if the test matches, and fails if + the test does not match. + """ + secrets = self.new_secrets() + storage_index = new_storage_index() + (written, _) = yield self.staraw( + storage_index, + secrets, + tw_vectors={ + 0: ([], [(0, b"1" * 7)], 7), + }, + r_vector=[], + ) + self.assertEqual(written, True) + + # Test matches, so write happens: + (written, _) = yield self.staraw( + storage_index, + secrets, + tw_vectors={ + 0: ([(0, 7, b"eq", b"1" * 7)], [(0, b"2" * 7)], 7), + }, + r_vector=[], + ) + self.assertEqual(written, True) + (_, reads) = yield self.staraw( + storage_index, + secrets, + tw_vectors={}, + r_vector=[(0, 7)], + ) + self.assertEqual(reads, {0: [b"2" * 7]}) + + # Test does not match, so write does not happen: + (written, _) = yield self.staraw( + storage_index, + secrets, + tw_vectors={ + 0: ([(0, 7, b"eq", b"1" * 7)], [(0, b"3" * 7)], 7), + }, + r_vector=[], + ) + self.assertEqual(written, False) + (_, reads) = yield self.staraw( + storage_index, + secrets, + tw_vectors={}, + r_vector=[(0, 7)], + ) + self.assertEqual(reads, {0: [b"2" * 7]}) + class _FoolscapMixin(SystemTestMixin): """Run tests on Foolscap version of ``IStorageServer.""" From 7b97ecfb7c27513f89129df0687da9df12e33b9f Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 14 Sep 2021 12:47:03 -0400 Subject: [PATCH 182/269] More tests. --- src/allmydata/test/test_istorageserver.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index ddb855c4f..796d6a9c6 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -277,8 +277,10 @@ class IStorageServerMutableAPIsTestsMixin(object): # DONE reads happen before (re)writes # DONE write happens if test succeeds # DONE write prevented if tests fail - # TODO partial reads, reads beyond the edge - # TODO wrong write enabled prevents writes + # TODO multiple test vectors + # TODO multiple writes + # TODO multiple reads + # TODO wrong write enabler secret prevents writes # TODO write prevented if test data against empty share # TODO writes can create additional shares if only some exist # TODO later writes overwrite @@ -292,7 +294,7 @@ class IStorageServerMutableAPIsTestsMixin(object): return self.storage_server.slot_testv_and_readv_and_writev(*args, **kwargs) @inlineCallbacks - def test_STARAW_reads(self): + def test_STARAW_reads_after_write(self): """ When data is written with ``IStorageServer.slot_testv_and_readv_and_writev``, it can then be read From 98e566fc44c5fde0b0e408e8ada65593762b33fd Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 14 Sep 2021 12:51:32 -0400 Subject: [PATCH 183/269] Expand testing scope. --- src/allmydata/test/test_istorageserver.py | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index 796d6a9c6..28b285151 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -277,9 +277,9 @@ class IStorageServerMutableAPIsTestsMixin(object): # DONE reads happen before (re)writes # DONE write happens if test succeeds # DONE write prevented if tests fail - # TODO multiple test vectors - # TODO multiple writes - # TODO multiple reads + # DONE multiple test vectors + # DONE multiple writes + # DONE multiple reads # TODO wrong write enabler secret prevents writes # TODO write prevented if test data against empty share # TODO writes can create additional shares if only some exist @@ -307,7 +307,7 @@ class IStorageServerMutableAPIsTestsMixin(object): secrets, tw_vectors={ 0: ([], [(0, b"abcdefg")], 7), - 1: ([], [(0, b"0123456")], 7), + 1: ([], [(0, b"0123"), (4, b"456")], 7), }, r_vector=[], ) @@ -317,9 +317,14 @@ class IStorageServerMutableAPIsTestsMixin(object): storage_index, secrets, tw_vectors={}, - r_vector=[(0, 7)], + # Whole thing, partial, going beyond the edge, completely outside + # range: + r_vector=[(0, 7), (2, 3), (6, 8), (100, 10)], + ) + self.assertEqual( + reads, + {0: [b"abcdefg", b"cde", b"g", b""], 1: [b"0123456", b"234", b"6", b""]}, ) - self.assertEqual(reads, {0: [b"abcdefg"], 1: [b"0123456"]}) @inlineCallbacks def test_SATRAW_reads_happen_before_writes_in_single_query(self): @@ -385,7 +390,11 @@ class IStorageServerMutableAPIsTestsMixin(object): storage_index, secrets, tw_vectors={ - 0: ([(0, 7, b"eq", b"1" * 7)], [(0, b"2" * 7)], 7), + 0: ( + [(0, 3, b"eq", b"1" * 3), (3, 4, b"eq", b"1" * 4)], + [(0, b"2" * 7)], + 7, + ), }, r_vector=[], ) From 241f4c841b33884adf301e2a7440d3948b990112 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 14 Sep 2021 13:00:29 -0400 Subject: [PATCH 184/269] Another test. --- src/allmydata/test/test_istorageserver.py | 42 +++++++++++++++++++++-- 1 file changed, 40 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index 28b285151..c299841f9 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -24,7 +24,7 @@ from testtools import skipIf from twisted.internet.defer import inlineCallbacks -from foolscap.api import Referenceable +from foolscap.api import Referenceable, RemoteException from allmydata.interfaces import IStorageServer from .common_system import SystemTestMixin @@ -280,7 +280,7 @@ class IStorageServerMutableAPIsTestsMixin(object): # DONE multiple test vectors # DONE multiple writes # DONE multiple reads - # TODO wrong write enabler secret prevents writes + # DONE wrong write enabler secret prevents writes # TODO write prevented if test data against empty share # TODO writes can create additional shares if only some exist # TODO later writes overwrite @@ -425,6 +425,44 @@ class IStorageServerMutableAPIsTestsMixin(object): ) self.assertEqual(reads, {0: [b"2" * 7]}) + @inlineCallbacks + def test_STARAW_write_enabler_must_match(self): + """ + If the write enabler secret passed to + ``IStorageServer.slot_testv_and_readv_and_writev`` doesn't match + previous writes, the write fails. + """ + secrets = self.new_secrets() + storage_index = new_storage_index() + (written, _) = yield self.staraw( + storage_index, + secrets, + tw_vectors={ + 0: ([], [(0, b"1" * 7)], 7), + }, + r_vector=[], + ) + self.assertEqual(written, True) + + # Write enabler secret does not match, so write does not happen: + bad_secrets = (new_secret(),) + secrets[1:] + with self.assertRaises(RemoteException): + yield self.staraw( + storage_index, + bad_secrets, + tw_vectors={ + 0: ([], [(0, b"2" * 7)], 7), + }, + r_vector=[], + ) + (_, reads) = yield self.staraw( + storage_index, + secrets, + tw_vectors={}, + r_vector=[(0, 7)], + ) + self.assertEqual(reads, {0: [b"1" * 7]}) + class _FoolscapMixin(SystemTestMixin): """Run tests on Foolscap version of ``IStorageServer.""" From 88a2e7a4fb993fb28a14a6cfe1ac1025a998bb93 Mon Sep 17 00:00:00 2001 From: Florian Sesser Date: Wed, 15 Sep 2021 10:09:55 +0000 Subject: [PATCH 185/269] OpenMetrics test suite: Get rid of status mock --- src/allmydata/test/test_openmetrics.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/test_openmetrics.py b/src/allmydata/test/test_openmetrics.py index fdb645b42..57ed989e0 100644 --- a/src/allmydata/test/test_openmetrics.py +++ b/src/allmydata/test/test_openmetrics.py @@ -96,6 +96,10 @@ class FakeStatsProvider(object): } return stats +class FakeStats(): + def __init__(self): + self._provider = FakeStatsProvider() + class OpenMetrics(unittest.TestCase): def test_spec_compliance(self): """ @@ -103,8 +107,7 @@ class OpenMetrics(unittest.TestCase): https://github.com/OpenObservability/OpenMetrics/blob/main/specification/OpenMetrics.md """ req = mock.Mock() - stats = mock.Mock() - stats._provider = FakeStatsProvider() + stats = FakeStats() metrics = Statistics.render_OPENMETRICS(stats, req) # "The content type MUST be..." From 57a3c1168e98fbce9b6b10d8a20100f4384a1620 Mon Sep 17 00:00:00 2001 From: Florian Sesser Date: Wed, 15 Sep 2021 11:03:31 +0000 Subject: [PATCH 186/269] OpenMetrics: Use list of strings instead of string concatenation --- src/allmydata/web/status.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/allmydata/web/status.py b/src/allmydata/web/status.py index 6d36861ad..2542ca75f 100644 --- a/src/allmydata/web/status.py +++ b/src/allmydata/web/status.py @@ -1556,7 +1556,7 @@ class Statistics(MultiFormatResource): def render_OPENMETRICS(self, req): req.setHeader("content-type", "application/openmetrics-text; version=1.0.0; charset=utf-8") stats = self._provider.get_stats() - ret = u"" + ret = [] def mangle_name(name): return re.sub( @@ -1569,13 +1569,13 @@ class Statistics(MultiFormatResource): return str(val) if val is not None else u"NaN" for (k, v) in sorted(stats['counters'].items()): - ret += u"tahoe_counters_%s %s\n" % (mangle_name(k), mangle_value(v)) + ret.append(u"tahoe_counters_%s %s" % (mangle_name(k), mangle_value(v))) for (k, v) in sorted(stats['stats'].items()): - ret += u"tahoe_stats_%s %s\n" % (mangle_name(k), mangle_value(v)) + ret.append(u"tahoe_stats_%s %s" % (mangle_name(k), mangle_value(v))) - ret += u"# EOF\n" + ret.append(u"# EOF") - return ret + return u"\n".join(ret) class StatisticsElement(Element): From d864cab5b0861a7c4a4caec294bafac884f318a6 Mon Sep 17 00:00:00 2001 From: Florian Sesser Date: Wed, 15 Sep 2021 11:14:52 +0000 Subject: [PATCH 187/269] OpenMetrics: Add test dep to nix packaging --- nix/tahoe-lafs.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/nix/tahoe-lafs.nix b/nix/tahoe-lafs.nix index 35b29f1cc..17b2f4463 100644 --- a/nix/tahoe-lafs.nix +++ b/nix/tahoe-lafs.nix @@ -107,6 +107,7 @@ EOF beautifulsoup4 html5lib tenacity + prometheus_client ]; checkPhase = '' From c66ae302c8412365c0f3848298351a1d69d975d8 Mon Sep 17 00:00:00 2001 From: Florian Sesser Date: Wed, 15 Sep 2021 11:26:30 +0000 Subject: [PATCH 188/269] OpenMetrics: Extra newline at the end --- src/allmydata/web/status.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/web/status.py b/src/allmydata/web/status.py index 2542ca75f..7b16a60b2 100644 --- a/src/allmydata/web/status.py +++ b/src/allmydata/web/status.py @@ -1573,7 +1573,7 @@ class Statistics(MultiFormatResource): for (k, v) in sorted(stats['stats'].items()): ret.append(u"tahoe_stats_%s %s" % (mangle_name(k), mangle_value(v))) - ret.append(u"# EOF") + ret.append(u"# EOF\n") return u"\n".join(ret) From cbe5ea1115e3e77e86d69a4dc2b29e508724e18e Mon Sep 17 00:00:00 2001 From: Florian Sesser Date: Wed, 15 Sep 2021 11:28:39 +0000 Subject: [PATCH 189/269] OpenMetrics: Add docstring --- src/allmydata/web/status.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/allmydata/web/status.py b/src/allmydata/web/status.py index 7b16a60b2..65647f491 100644 --- a/src/allmydata/web/status.py +++ b/src/allmydata/web/status.py @@ -1554,6 +1554,12 @@ class Statistics(MultiFormatResource): @render_exception def render_OPENMETRICS(self, req): + """ + Render our stats in `OpenMetrics ` format. + For example Prometheus and Victoriametrics can parse this. + Point the scraper to ``/statistics?t=openmetrics`` (instead of the + default ``/metrics``). + """ req.setHeader("content-type", "application/openmetrics-text; version=1.0.0; charset=utf-8") stats = self._provider.get_stats() ret = [] From 21c471ed8113ce29fd1b6c6dc02e2acf124d550a Mon Sep 17 00:00:00 2001 From: Florian Sesser Date: Wed, 15 Sep 2021 11:39:32 +0000 Subject: [PATCH 190/269] OpenMetrics test: Add hopefully more stable URIs to OpenMetrics spec info --- src/allmydata/test/test_openmetrics.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/test_openmetrics.py b/src/allmydata/test/test_openmetrics.py index 57ed989e0..7753960d4 100644 --- a/src/allmydata/test/test_openmetrics.py +++ b/src/allmydata/test/test_openmetrics.py @@ -103,8 +103,9 @@ class FakeStats(): class OpenMetrics(unittest.TestCase): def test_spec_compliance(self): """ - Does our output adhere to the OpenMetrics spec? - https://github.com/OpenObservability/OpenMetrics/blob/main/specification/OpenMetrics.md + Does our output adhere to the `OpenMetrics ` spec? + https://github.com/OpenObservability/OpenMetrics/ + https://prometheus.io/docs/instrumenting/exposition_formats/ """ req = mock.Mock() stats = FakeStats() From 6bcff5472b9aca6dd09e585e80bb937832d254e7 Mon Sep 17 00:00:00 2001 From: Florian Sesser Date: Wed, 15 Sep 2021 11:50:20 +0000 Subject: [PATCH 191/269] OpenMetrics test suite: Add a check to see whether our stats were parsed at all. --- src/allmydata/test/test_openmetrics.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/allmydata/test/test_openmetrics.py b/src/allmydata/test/test_openmetrics.py index 7753960d4..a9a0e5712 100644 --- a/src/allmydata/test/test_openmetrics.py +++ b/src/allmydata/test/test_openmetrics.py @@ -120,3 +120,7 @@ class OpenMetrics(unittest.TestCase): # Wrapped in a list() to drain the generator. families = list(parser.text_string_to_metric_families(metrics)) + # Has the parser parsed our data? + # Just check the last item. + self.assertEqual(families[-1].name, u"tahoe_stats_storage_server_total_bucket_count") + From 383ab4729a7c362f25526097c3f6c66e9451ef2c Mon Sep 17 00:00:00 2001 From: Florian Sesser Date: Wed, 15 Sep 2021 11:53:48 +0000 Subject: [PATCH 192/269] OpenMetrics tests: Tryfix resolve TypeError on CI Was: > TypeError: unbound method render_OPENMETRICS() must be called with Statistics instance as first argument (got FakeStats instance instead) --- src/allmydata/test/test_openmetrics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/test_openmetrics.py b/src/allmydata/test/test_openmetrics.py index a9a0e5712..9c840714c 100644 --- a/src/allmydata/test/test_openmetrics.py +++ b/src/allmydata/test/test_openmetrics.py @@ -96,7 +96,7 @@ class FakeStatsProvider(object): } return stats -class FakeStats(): +class FakeStats(Statistics): def __init__(self): self._provider = FakeStatsProvider() From d210062dd7e943c7038bb312709a21c2196d5f86 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 15 Sep 2021 09:47:16 -0400 Subject: [PATCH 193/269] Another test for STARAW. --- src/allmydata/test/test_istorageserver.py | 52 +++++++++++++++++------ 1 file changed, 38 insertions(+), 14 deletions(-) diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index c299841f9..8145bef44 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -271,20 +271,6 @@ class IStorageServerMutableAPIsTestsMixin(object): ``STARAW`` is short for ``slot_testv_and_readv_and_writev``. """ - - # slot_testv_and_readv_and_writev - # DONE it's possible to write and then in separate call read - # DONE reads happen before (re)writes - # DONE write happens if test succeeds - # DONE write prevented if tests fail - # DONE multiple test vectors - # DONE multiple writes - # DONE multiple reads - # DONE wrong write enabler secret prevents writes - # TODO write prevented if test data against empty share - # TODO writes can create additional shares if only some exist - # TODO later writes overwrite - def new_secrets(self): """Return a 3-tuple of secrets for STARAW calls.""" return (new_secret(), new_secret(), new_secret()) @@ -463,6 +449,44 @@ class IStorageServerMutableAPIsTestsMixin(object): ) self.assertEqual(reads, {0: [b"1" * 7]}) + @inlineCallbacks + def test_STARAW_zero_new_length_deletes(self): + """ + A zero new length passed to + ``IStorageServer.slot_testv_and_readv_and_writev`` deletes the share. + """ + secrets = self.new_secrets() + storage_index = new_storage_index() + (written, _) = yield self.staraw( + storage_index, + secrets, + tw_vectors={ + 0: ([], [(0, b"1" * 7)], 7), + }, + r_vector=[], + ) + self.assertEqual(written, True) + + # Write with new length of 0: + (written, _) = yield self.staraw( + storage_index, + secrets, + tw_vectors={ + 0: ([], [(0, b"1" * 7)], 0), + }, + r_vector=[], + ) + self.assertEqual(written, True) + + # It's gone! + (_, reads) = yield self.staraw( + storage_index, + secrets, + tw_vectors={}, + r_vector=[(0, 7)], + ) + self.assertEqual(reads, {}) + class _FoolscapMixin(SystemTestMixin): """Run tests on Foolscap version of ``IStorageServer.""" From 863343298060a9bfca1cc95bda0f4ca88325be1c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 15 Sep 2021 10:33:51 -0400 Subject: [PATCH 194/269] Switch IStorageServer interface to be slightly different than RIStorageServer. --- src/allmydata/interfaces.py | 4 ++++ src/allmydata/mutable/layout.py | 12 ++++++------ src/allmydata/storage_client.py | 10 +++++++++- 3 files changed, 19 insertions(+), 7 deletions(-) diff --git a/src/allmydata/interfaces.py b/src/allmydata/interfaces.py index 1c64bce8a..9000787e8 100644 --- a/src/allmydata/interfaces.py +++ b/src/allmydata/interfaces.py @@ -351,6 +351,10 @@ class IStorageServer(Interface): ): """ :see: ``RIStorageServer.slot_testv_readv_and_writev`` + + While the interface mostly matches, test vectors are simplified. + Instead of a tuple ``(start, offset, operator, data)`` they are just + ``(start, data)`` with operator implicitly being ``b"eq"``. """ def advise_corrupt_share( diff --git a/src/allmydata/mutable/layout.py b/src/allmydata/mutable/layout.py index ce51a8833..50b60e30c 100644 --- a/src/allmydata/mutable/layout.py +++ b/src/allmydata/mutable/layout.py @@ -309,7 +309,7 @@ class SDMFSlotWriteProxy(object): salt) else: checkstring = checkstring_or_seqnum - self._testvs = [(0, len(checkstring), b"eq", checkstring)] + self._testvs = [(0, checkstring)] def get_checkstring(self): @@ -318,7 +318,7 @@ class SDMFSlotWriteProxy(object): server. """ if self._testvs: - return self._testvs[0][3] + return self._testvs[0][1] return b"" @@ -550,7 +550,7 @@ class SDMFSlotWriteProxy(object): # yet, so we assume that we are writing a new share, and set # a test vector that will allow a new share to be written. self._testvs = [] - self._testvs.append(tuple([0, 1, b"eq", b""])) + self._testvs.append(tuple([0, b""])) tw_vectors = {} tw_vectors[self.shnum] = (self._testvs, datavs, None) @@ -889,7 +889,7 @@ class MDMFSlotWriteProxy(object): self._testvs = [] else: self._testvs = [] - self._testvs.append((0, len(checkstring), b"eq", checkstring)) + self._testvs.append((0, checkstring)) def __repr__(self): @@ -1162,7 +1162,7 @@ class MDMFSlotWriteProxy(object): tw_vectors = {} if not self._testvs: self._testvs = [] - self._testvs.append(tuple([0, 1, b"eq", b""])) + self._testvs.append(tuple([0, b""])) if not self._written: # Write a new checkstring to the share when we write it, so # that we have something to check later. @@ -1170,7 +1170,7 @@ class MDMFSlotWriteProxy(object): datavs.append((0, new_checkstring)) def _first_write(): self._written = True - self._testvs = [(0, len(new_checkstring), b"eq", new_checkstring)] + self._testvs = [(0, new_checkstring)] on_success = _first_write tw_vectors[self.shnum] = (self._testvs, datavs, None) d = self._storage_server.slot_testv_and_readv_and_writev( diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index e9dc8c84c..ec877779d 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -994,11 +994,19 @@ class _StorageServer(object): tw_vectors, r_vector, ): + # Match the wire protocol, which requires 4-tuples for test vectors. + wire_format_tw_vectors = { + key: ( + [(start, len(data), b"eq", data) for (start, data) in value[0]], + value[1], + value[2], + ) for (key, value) in tw_vectors.items() + } return self._rref.callRemote( "slot_testv_and_readv_and_writev", storage_index, secrets, - tw_vectors, + wire_format_tw_vectors, r_vector, ) From f109afa3b130ab9c89209f794e8309cf1b5ab917 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 15 Sep 2021 10:51:43 -0400 Subject: [PATCH 195/269] This is unnecessary, empty vector list is fine too. --- src/allmydata/mutable/layout.py | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/src/allmydata/mutable/layout.py b/src/allmydata/mutable/layout.py index 50b60e30c..35f932b89 100644 --- a/src/allmydata/mutable/layout.py +++ b/src/allmydata/mutable/layout.py @@ -545,13 +545,6 @@ class SDMFSlotWriteProxy(object): # in its entirely. datavs = [(0, final_share)] - if not self._testvs: - # Our caller has not provided us with another checkstring - # yet, so we assume that we are writing a new share, and set - # a test vector that will allow a new share to be written. - self._testvs = [] - self._testvs.append(tuple([0, b""])) - tw_vectors = {} tw_vectors[self.shnum] = (self._testvs, datavs, None) return self._storage_server.slot_testv_and_readv_and_writev( @@ -888,8 +881,7 @@ class MDMFSlotWriteProxy(object): # empty string means. self._testvs = [] else: - self._testvs = [] - self._testvs.append((0, checkstring)) + self._testvs = [(0, checkstring)] def __repr__(self): @@ -1160,9 +1152,6 @@ class MDMFSlotWriteProxy(object): def _write(self, datavs, on_failure=None, on_success=None): """I write the data vectors in datavs to the remote slot.""" tw_vectors = {} - if not self._testvs: - self._testvs = [] - self._testvs.append(tuple([0, b""])) if not self._written: # Write a new checkstring to the share when we write it, so # that we have something to check later. From 911a5e2ed1dfb6ffc593a6d402d98f3783d2397f Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 15 Sep 2021 11:07:02 -0400 Subject: [PATCH 196/269] Rip out server-side usage of operators other than eq, because nothing ever used them. --- src/allmydata/interfaces.py | 5 +- src/allmydata/storage/mutable.py | 17 +--- src/allmydata/test/mutable/util.py | 2 +- src/allmydata/test/test_storage.py | 149 ----------------------------- 4 files changed, 6 insertions(+), 167 deletions(-) diff --git a/src/allmydata/interfaces.py b/src/allmydata/interfaces.py index 9000787e8..dcd442624 100644 --- a/src/allmydata/interfaces.py +++ b/src/allmydata/interfaces.py @@ -91,9 +91,8 @@ class RIBucketReader(RemoteInterface): TestVector = ListOf(TupleOf(Offset, ReadSize, bytes, bytes)) # elements are (offset, length, operator, specimen) -# operator is one of "lt, le, eq, ne, ge, gt" -# nop always passes and is used to fetch data while writing. -# you should use length==len(specimen) for everything except nop +# operator must be b"eq", you should use length==len(specimen). +# (These are only used for wire compatibility with old versions). DataVector = ListOf(TupleOf(Offset, ShareData)) # (offset, data). This limits us to 30 writes of 1MiB each per call TestAndWriteVectorsForShares = DictOf(int, diff --git a/src/allmydata/storage/mutable.py b/src/allmydata/storage/mutable.py index a44a2e18d..2ef0c3215 100644 --- a/src/allmydata/storage/mutable.py +++ b/src/allmydata/storage/mutable.py @@ -434,20 +434,9 @@ class MutableShareFile(object): # self._change_container_size() here. def testv_compare(a, op, b): - assert op in (b"lt", b"le", b"eq", b"ne", b"ge", b"gt") - if op == b"lt": - return a < b - if op == b"le": - return a <= b - if op == b"eq": - return a == b - if op == b"ne": - return a != b - if op == b"ge": - return a >= b - if op == b"gt": - return a > b - # never reached + assert op == b"eq" + return a == b + class EmptyShare(object): diff --git a/src/allmydata/test/mutable/util.py b/src/allmydata/test/mutable/util.py index 7e3bd3ec7..dac61a6e3 100644 --- a/src/allmydata/test/mutable/util.py +++ b/src/allmydata/test/mutable/util.py @@ -149,7 +149,7 @@ class FakeStorageServer(object): readv = {} for shnum, (testv, writev, new_length) in list(tw_vectors.items()): for (offset, length, op, specimen) in testv: - assert op in (b"le", b"eq", b"ge") + assert op == b"eq" # TODO: this isn't right, the read is controlled by read_vector, # not by testv readv[shnum] = [ specimen diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index bd0ab80f3..42a3ed0a1 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -1074,23 +1074,6 @@ class MutableServer(unittest.TestCase): })) self.failUnlessEqual(read(b"si1", [0], [(0,100)]), {0: [data]}) - # as should this one - answer = write(b"si1", secrets, - {0: ([(10, 5, b"lt", b"11111"), - ], - [(0, b"x"*100)], - None), - }, - [(10,5)], - ) - self.failUnlessEqual(answer, (False, - {0: [b"11111"], - 1: [b""], - 2: [b""]}, - )) - self.failUnlessEqual(read(b"si1", [0], [(0,100)]), {0: [data]}) - - def test_operators(self): # test operators, the data we're comparing is '11111' in all cases. # test both fail+pass, reset data after each one. @@ -1110,63 +1093,6 @@ class MutableServer(unittest.TestCase): reset() - # lt - answer = write(b"si1", secrets, {0: ([(10, 5, b"lt", b"11110"), - ], - [(0, b"x"*100)], - None, - )}, [(10,5)]) - self.failUnlessEqual(answer, (False, {0: [b"11111"]})) - self.failUnlessEqual(read(b"si1", [0], [(0,100)]), {0: [data]}) - self.failUnlessEqual(read(b"si1", [], [(0,100)]), {0: [data]}) - reset() - - answer = write(b"si1", secrets, {0: ([(10, 5, b"lt", b"11111"), - ], - [(0, b"x"*100)], - None, - )}, [(10,5)]) - self.failUnlessEqual(answer, (False, {0: [b"11111"]})) - self.failUnlessEqual(read(b"si1", [0], [(0,100)]), {0: [data]}) - reset() - - answer = write(b"si1", secrets, {0: ([(10, 5, b"lt", b"11112"), - ], - [(0, b"y"*100)], - None, - )}, [(10,5)]) - self.failUnlessEqual(answer, (True, {0: [b"11111"]})) - self.failUnlessEqual(read(b"si1", [0], [(0,100)]), {0: [b"y"*100]}) - reset() - - # le - answer = write(b"si1", secrets, {0: ([(10, 5, b"le", b"11110"), - ], - [(0, b"x"*100)], - None, - )}, [(10,5)]) - self.failUnlessEqual(answer, (False, {0: [b"11111"]})) - self.failUnlessEqual(read(b"si1", [0], [(0,100)]), {0: [data]}) - reset() - - answer = write(b"si1", secrets, {0: ([(10, 5, b"le", b"11111"), - ], - [(0, b"y"*100)], - None, - )}, [(10,5)]) - self.failUnlessEqual(answer, (True, {0: [b"11111"]})) - self.failUnlessEqual(read(b"si1", [0], [(0,100)]), {0: [b"y"*100]}) - reset() - - answer = write(b"si1", secrets, {0: ([(10, 5, b"le", b"11112"), - ], - [(0, b"y"*100)], - None, - )}, [(10,5)]) - self.failUnlessEqual(answer, (True, {0: [b"11111"]})) - self.failUnlessEqual(read(b"si1", [0], [(0,100)]), {0: [b"y"*100]}) - reset() - # eq answer = write(b"si1", secrets, {0: ([(10, 5, b"eq", b"11112"), ], @@ -1186,81 +1112,6 @@ class MutableServer(unittest.TestCase): self.failUnlessEqual(read(b"si1", [0], [(0,100)]), {0: [b"y"*100]}) reset() - # ne - answer = write(b"si1", secrets, {0: ([(10, 5, b"ne", b"11111"), - ], - [(0, b"x"*100)], - None, - )}, [(10,5)]) - self.failUnlessEqual(answer, (False, {0: [b"11111"]})) - self.failUnlessEqual(read(b"si1", [0], [(0,100)]), {0: [data]}) - reset() - - answer = write(b"si1", secrets, {0: ([(10, 5, b"ne", b"11112"), - ], - [(0, b"y"*100)], - None, - )}, [(10,5)]) - self.failUnlessEqual(answer, (True, {0: [b"11111"]})) - self.failUnlessEqual(read(b"si1", [0], [(0,100)]), {0: [b"y"*100]}) - reset() - - # ge - answer = write(b"si1", secrets, {0: ([(10, 5, b"ge", b"11110"), - ], - [(0, b"y"*100)], - None, - )}, [(10,5)]) - self.failUnlessEqual(answer, (True, {0: [b"11111"]})) - self.failUnlessEqual(read(b"si1", [0], [(0,100)]), {0: [b"y"*100]}) - reset() - - answer = write(b"si1", secrets, {0: ([(10, 5, b"ge", b"11111"), - ], - [(0, b"y"*100)], - None, - )}, [(10,5)]) - self.failUnlessEqual(answer, (True, {0: [b"11111"]})) - self.failUnlessEqual(read(b"si1", [0], [(0,100)]), {0: [b"y"*100]}) - reset() - - answer = write(b"si1", secrets, {0: ([(10, 5, b"ge", b"11112"), - ], - [(0, b"y"*100)], - None, - )}, [(10,5)]) - self.failUnlessEqual(answer, (False, {0: [b"11111"]})) - self.failUnlessEqual(read(b"si1", [0], [(0,100)]), {0: [data]}) - reset() - - # gt - answer = write(b"si1", secrets, {0: ([(10, 5, b"gt", b"11110"), - ], - [(0, b"y"*100)], - None, - )}, [(10,5)]) - self.failUnlessEqual(answer, (True, {0: [b"11111"]})) - self.failUnlessEqual(read(b"si1", [0], [(0,100)]), {0: [b"y"*100]}) - reset() - - answer = write(b"si1", secrets, {0: ([(10, 5, b"gt", b"11111"), - ], - [(0, b"x"*100)], - None, - )}, [(10,5)]) - self.failUnlessEqual(answer, (False, {0: [b"11111"]})) - self.failUnlessEqual(read(b"si1", [0], [(0,100)]), {0: [data]}) - reset() - - answer = write(b"si1", secrets, {0: ([(10, 5, b"gt", b"11112"), - ], - [(0, b"x"*100)], - None, - )}, [(10,5)]) - self.failUnlessEqual(answer, (False, {0: [b"11111"]})) - self.failUnlessEqual(read(b"si1", [0], [(0,100)]), {0: [data]}) - reset() - # finally, test some operators against empty shares answer = write(b"si1", secrets, {1: ([(10, 5, b"eq", b"11112"), ], From 24c483aeda8c32233daee90baf02b0e378d4f464 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 15 Sep 2021 11:07:23 -0400 Subject: [PATCH 197/269] Update to simplified test vector support. --- docs/proposed/http-storage-node-protocol.rst | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index a84d62176..febcb250e 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -599,8 +599,6 @@ For example:: 0: { "test": [{ "offset": 3, - "size": 5, - "operator": "eq", "specimen": "hello" }, ...], "write": [{ @@ -712,12 +710,7 @@ Mutable Data }, "test-write-vectors": { 3: { - "test": [{ - "offset": 0, - "size": 1, - "operator": "eq", - "specimen": "" - }], + "test": [], "write": [{ "offset": 0, "data": "xxxxxxxxxx" @@ -747,8 +740,6 @@ Mutable Data 3: { "test": [{ "offset": 0, - "size": , - "operator": "eq", "specimen": "" }], "write": [{ From d9151b643a2cea518c294cd0c6c19b22b39ebfac Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 15 Sep 2021 11:07:54 -0400 Subject: [PATCH 198/269] News file. --- newsfragments/3799.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3799.minor diff --git a/newsfragments/3799.minor b/newsfragments/3799.minor new file mode 100644 index 000000000..e69de29bb From b0e1cf924d44b271314efac94106247afdc7be10 Mon Sep 17 00:00:00 2001 From: Florian Sesser Date: Wed, 15 Sep 2021 15:14:29 +0000 Subject: [PATCH 199/269] OpenMetrics test: White space only: Format JSON fixture to be easier on the eyes --- src/allmydata/test/test_openmetrics.py | 177 +++++++++++++------------ 1 file changed, 89 insertions(+), 88 deletions(-) diff --git a/src/allmydata/test/test_openmetrics.py b/src/allmydata/test/test_openmetrics.py index 9c840714c..7d9bd6429 100644 --- a/src/allmydata/test/test_openmetrics.py +++ b/src/allmydata/test/test_openmetrics.py @@ -6,94 +6,95 @@ from allmydata.web.status import Statistics class FakeStatsProvider(object): def get_stats(self): # Parsed into a dict from a running tahoe's /statistics?t=json - stats = {'stats': { - 'storage_server.latencies.get.99_9_percentile': None, - 'storage_server.latencies.close.10_0_percentile': 0.00021910667419433594, - 'storage_server.latencies.read.01_0_percentile': 2.8848648071289062e-05, - 'storage_server.latencies.writev.99_9_percentile': None, - 'storage_server.latencies.read.99_9_percentile': None, - 'storage_server.latencies.allocate.99_0_percentile': 0.000988006591796875, - 'storage_server.latencies.writev.mean': 0.00045332245070571654, - 'storage_server.latencies.close.99_9_percentile': None, - 'cpu_monitor.15min_avg': 0.00017592000079223033, - 'storage_server.disk_free_for_root': 103289454592, - 'storage_server.latencies.get.99_0_percentile': 0.000347137451171875, - 'storage_server.latencies.get.mean': 0.00021158285060171353, - 'storage_server.latencies.read.90_0_percentile': 8.893013000488281e-05, - 'storage_server.latencies.write.01_0_percentile': 3.600120544433594e-05, - 'storage_server.latencies.write.99_9_percentile': 0.00017690658569335938, - 'storage_server.latencies.close.90_0_percentile': 0.00033211708068847656, - 'storage_server.disk_total': 103497859072, - 'storage_server.latencies.close.95_0_percentile': 0.0003509521484375, - 'storage_server.latencies.readv.samplesize': 1000, - 'storage_server.disk_free_for_nonroot': 103289454592, - 'storage_server.latencies.close.mean': 0.0002715024480059103, - 'storage_server.latencies.writev.95_0_percentile': 0.0007410049438476562, - 'storage_server.latencies.readv.90_0_percentile': 0.0003781318664550781, - 'storage_server.latencies.readv.99_0_percentile': 0.0004050731658935547, - 'storage_server.latencies.allocate.mean': 0.0007128627429454784, - 'storage_server.latencies.close.samplesize': 326, - 'storage_server.latencies.get.50_0_percentile': 0.0001819133758544922, - 'storage_server.latencies.write.50_0_percentile': 4.482269287109375e-05, - 'storage_server.latencies.readv.01_0_percentile': 0.0002970695495605469, - 'storage_server.latencies.get.10_0_percentile': 0.00015687942504882812, - 'storage_server.latencies.allocate.90_0_percentile': 0.0008189678192138672, - 'storage_server.latencies.get.samplesize': 472, - 'storage_server.total_bucket_count': 393, - 'storage_server.latencies.read.mean': 5.936201880959903e-05, - 'storage_server.latencies.allocate.01_0_percentile': 0.0004208087921142578, - 'storage_server.latencies.allocate.99_9_percentile': None, - 'storage_server.latencies.readv.mean': 0.00034061360359191893, - 'storage_server.disk_used': 208404480, - 'storage_server.latencies.allocate.50_0_percentile': 0.0007410049438476562, - 'storage_server.latencies.read.99_0_percentile': 0.00011992454528808594, - 'node.uptime': 3805759.8545179367, - 'storage_server.latencies.writev.10_0_percentile': 0.00035190582275390625, - 'storage_server.latencies.writev.90_0_percentile': 0.0006821155548095703, - 'storage_server.latencies.close.01_0_percentile': 0.00021505355834960938, - 'storage_server.latencies.close.50_0_percentile': 0.0002579689025878906, - 'cpu_monitor.1min_avg': 0.0002130000000003444, - 'storage_server.latencies.writev.50_0_percentile': 0.0004138946533203125, - 'storage_server.latencies.read.95_0_percentile': 9.107589721679688e-05, - 'storage_server.latencies.readv.95_0_percentile': 0.0003859996795654297, - 'storage_server.latencies.write.10_0_percentile': 3.719329833984375e-05, - 'storage_server.accepting_immutable_shares': 1, - 'storage_server.latencies.writev.samplesize': 309, - 'storage_server.latencies.get.95_0_percentile': 0.0003190040588378906, - 'storage_server.latencies.readv.10_0_percentile': 0.00032210350036621094, - 'storage_server.latencies.get.90_0_percentile': 0.0002999305725097656, - 'storage_server.latencies.get.01_0_percentile': 0.0001239776611328125, - 'cpu_monitor.total': 641.4941180000001, - 'storage_server.latencies.write.samplesize': 1000, - 'storage_server.latencies.write.95_0_percentile': 9.489059448242188e-05, - 'storage_server.latencies.read.50_0_percentile': 6.890296936035156e-05, - 'storage_server.latencies.writev.01_0_percentile': 0.00033211708068847656, - 'storage_server.latencies.read.10_0_percentile': 3.0994415283203125e-05, - 'storage_server.latencies.allocate.10_0_percentile': 0.0004949569702148438, - 'storage_server.reserved_space': 0, - 'storage_server.disk_avail': 103289454592, - 'storage_server.latencies.write.99_0_percentile': 0.00011301040649414062, - 'storage_server.latencies.write.90_0_percentile': 9.083747863769531e-05, - 'cpu_monitor.5min_avg': 0.0002370666691157502, - 'storage_server.latencies.write.mean': 5.8008909225463864e-05, - 'storage_server.latencies.readv.50_0_percentile': 0.00033020973205566406, - 'storage_server.latencies.close.99_0_percentile': 0.0004038810729980469, - 'storage_server.allocated': 0, - 'storage_server.latencies.writev.99_0_percentile': 0.0007710456848144531, - 'storage_server.latencies.readv.99_9_percentile': 0.0004780292510986328, - 'storage_server.latencies.read.samplesize': 170, - 'storage_server.latencies.allocate.samplesize': 406, - 'storage_server.latencies.allocate.95_0_percentile': 0.0008411407470703125 - }, 'counters': { - 'storage_server.writev': 309, - 'storage_server.bytes_added': 197836146, - 'storage_server.close': 326, - 'storage_server.readv': 14299, - 'storage_server.allocate': 406, - 'storage_server.read': 170, - 'storage_server.write': 3775, - 'storage_server.get': 472} - } + stats = { + 'stats': { + 'storage_server.latencies.get.99_9_percentile': None, + 'storage_server.latencies.close.10_0_percentile': 0.00021910667419433594, + 'storage_server.latencies.read.01_0_percentile': 2.8848648071289062e-05, + 'storage_server.latencies.writev.99_9_percentile': None, + 'storage_server.latencies.read.99_9_percentile': None, + 'storage_server.latencies.allocate.99_0_percentile': 0.000988006591796875, + 'storage_server.latencies.writev.mean': 0.00045332245070571654, + 'storage_server.latencies.close.99_9_percentile': None, + 'cpu_monitor.15min_avg': 0.00017592000079223033, + 'storage_server.disk_free_for_root': 103289454592, + 'storage_server.latencies.get.99_0_percentile': 0.000347137451171875, + 'storage_server.latencies.get.mean': 0.00021158285060171353, + 'storage_server.latencies.read.90_0_percentile': 8.893013000488281e-05, + 'storage_server.latencies.write.01_0_percentile': 3.600120544433594e-05, + 'storage_server.latencies.write.99_9_percentile': 0.00017690658569335938, + 'storage_server.latencies.close.90_0_percentile': 0.00033211708068847656, + 'storage_server.disk_total': 103497859072, + 'storage_server.latencies.close.95_0_percentile': 0.0003509521484375, + 'storage_server.latencies.readv.samplesize': 1000, + 'storage_server.disk_free_for_nonroot': 103289454592, + 'storage_server.latencies.close.mean': 0.0002715024480059103, + 'storage_server.latencies.writev.95_0_percentile': 0.0007410049438476562, + 'storage_server.latencies.readv.90_0_percentile': 0.0003781318664550781, + 'storage_server.latencies.readv.99_0_percentile': 0.0004050731658935547, + 'storage_server.latencies.allocate.mean': 0.0007128627429454784, + 'storage_server.latencies.close.samplesize': 326, + 'storage_server.latencies.get.50_0_percentile': 0.0001819133758544922, + 'storage_server.latencies.write.50_0_percentile': 4.482269287109375e-05, + 'storage_server.latencies.readv.01_0_percentile': 0.0002970695495605469, + 'storage_server.latencies.get.10_0_percentile': 0.00015687942504882812, + 'storage_server.latencies.allocate.90_0_percentile': 0.0008189678192138672, + 'storage_server.latencies.get.samplesize': 472, + 'storage_server.total_bucket_count': 393, + 'storage_server.latencies.read.mean': 5.936201880959903e-05, + 'storage_server.latencies.allocate.01_0_percentile': 0.0004208087921142578, + 'storage_server.latencies.allocate.99_9_percentile': None, + 'storage_server.latencies.readv.mean': 0.00034061360359191893, + 'storage_server.disk_used': 208404480, + 'storage_server.latencies.allocate.50_0_percentile': 0.0007410049438476562, + 'storage_server.latencies.read.99_0_percentile': 0.00011992454528808594, + 'node.uptime': 3805759.8545179367, + 'storage_server.latencies.writev.10_0_percentile': 0.00035190582275390625, + 'storage_server.latencies.writev.90_0_percentile': 0.0006821155548095703, + 'storage_server.latencies.close.01_0_percentile': 0.00021505355834960938, + 'storage_server.latencies.close.50_0_percentile': 0.0002579689025878906, + 'cpu_monitor.1min_avg': 0.0002130000000003444, + 'storage_server.latencies.writev.50_0_percentile': 0.0004138946533203125, + 'storage_server.latencies.read.95_0_percentile': 9.107589721679688e-05, + 'storage_server.latencies.readv.95_0_percentile': 0.0003859996795654297, + 'storage_server.latencies.write.10_0_percentile': 3.719329833984375e-05, + 'storage_server.accepting_immutable_shares': 1, + 'storage_server.latencies.writev.samplesize': 309, + 'storage_server.latencies.get.95_0_percentile': 0.0003190040588378906, + 'storage_server.latencies.readv.10_0_percentile': 0.00032210350036621094, + 'storage_server.latencies.get.90_0_percentile': 0.0002999305725097656, + 'storage_server.latencies.get.01_0_percentile': 0.0001239776611328125, + 'cpu_monitor.total': 641.4941180000001, + 'storage_server.latencies.write.samplesize': 1000, + 'storage_server.latencies.write.95_0_percentile': 9.489059448242188e-05, + 'storage_server.latencies.read.50_0_percentile': 6.890296936035156e-05, + 'storage_server.latencies.writev.01_0_percentile': 0.00033211708068847656, + 'storage_server.latencies.read.10_0_percentile': 3.0994415283203125e-05, + 'storage_server.latencies.allocate.10_0_percentile': 0.0004949569702148438, + 'storage_server.reserved_space': 0, + 'storage_server.disk_avail': 103289454592, + 'storage_server.latencies.write.99_0_percentile': 0.00011301040649414062, + 'storage_server.latencies.write.90_0_percentile': 9.083747863769531e-05, + 'cpu_monitor.5min_avg': 0.0002370666691157502, + 'storage_server.latencies.write.mean': 5.8008909225463864e-05, + 'storage_server.latencies.readv.50_0_percentile': 0.00033020973205566406, + 'storage_server.latencies.close.99_0_percentile': 0.0004038810729980469, + 'storage_server.allocated': 0, + 'storage_server.latencies.writev.99_0_percentile': 0.0007710456848144531, + 'storage_server.latencies.readv.99_9_percentile': 0.0004780292510986328, + 'storage_server.latencies.read.samplesize': 170, + 'storage_server.latencies.allocate.samplesize': 406, + 'storage_server.latencies.allocate.95_0_percentile': 0.0008411407470703125}, + 'counters': { + 'storage_server.writev': 309, + 'storage_server.bytes_added': 197836146, + 'storage_server.close': 326, + 'storage_server.readv': 14299, + 'storage_server.allocate': 406, + 'storage_server.read': 170, + 'storage_server.write': 3775, + 'storage_server.get': 472} + } return stats class FakeStats(Statistics): From 1d2073b8f8bd883a364ba4b18e3c6cfeb8e68fd7 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 15 Sep 2021 16:19:24 -0400 Subject: [PATCH 200/269] Revert "This is unnecessary, empty vector list is fine too." This reverts commit f109afa3b130ab9c89209f794e8309cf1b5ab917. --- src/allmydata/mutable/layout.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/allmydata/mutable/layout.py b/src/allmydata/mutable/layout.py index 35f932b89..50b60e30c 100644 --- a/src/allmydata/mutable/layout.py +++ b/src/allmydata/mutable/layout.py @@ -545,6 +545,13 @@ class SDMFSlotWriteProxy(object): # in its entirely. datavs = [(0, final_share)] + if not self._testvs: + # Our caller has not provided us with another checkstring + # yet, so we assume that we are writing a new share, and set + # a test vector that will allow a new share to be written. + self._testvs = [] + self._testvs.append(tuple([0, b""])) + tw_vectors = {} tw_vectors[self.shnum] = (self._testvs, datavs, None) return self._storage_server.slot_testv_and_readv_and_writev( @@ -881,7 +888,8 @@ class MDMFSlotWriteProxy(object): # empty string means. self._testvs = [] else: - self._testvs = [(0, checkstring)] + self._testvs = [] + self._testvs.append((0, checkstring)) def __repr__(self): @@ -1152,6 +1160,9 @@ class MDMFSlotWriteProxy(object): def _write(self, datavs, on_failure=None, on_success=None): """I write the data vectors in datavs to the remote slot.""" tw_vectors = {} + if not self._testvs: + self._testvs = [] + self._testvs.append(tuple([0, b""])) if not self._written: # Write a new checkstring to the share when we write it, so # that we have something to check later. From e11e5dfbe6c5a8556d5d29e187f4a8cb0e899931 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 15 Sep 2021 16:31:54 -0400 Subject: [PATCH 201/269] Revert removal of length in IStorageServer. --- docs/proposed/http-storage-node-protocol.rst | 13 +++++++++++-- src/allmydata/interfaces.py | 7 ++++--- src/allmydata/mutable/layout.py | 16 +++++++++------- src/allmydata/storage_client.py | 2 +- 4 files changed, 25 insertions(+), 13 deletions(-) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index febcb250e..35811cad0 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -599,6 +599,7 @@ For example:: 0: { "test": [{ "offset": 3, + "size": 5, "specimen": "hello" }, ...], "write": [{ @@ -699,7 +700,10 @@ Immutable Data Mutable Data ~~~~~~~~~~~~ -1. Create mutable share number ``3`` with ``10`` bytes of data in slot ``BBBBBBBBBBBBBBBB``:: +1. Create mutable share number ``3`` with ``10`` bytes of data in slot ``BBBBBBBBBBBBBBBB``. +The special test vector of size 1 but empty bytes will only pass +if there is no existing share, +otherwise it will read a byte which won't match `b""`:: POST /v1/mutable/BBBBBBBBBBBBBBBB/read-test-write { @@ -710,7 +714,11 @@ Mutable Data }, "test-write-vectors": { 3: { - "test": [], + "test": [{ + "offset": 0, + "size": 1, + "specimen": "" + }], "write": [{ "offset": 0, "data": "xxxxxxxxxx" @@ -740,6 +748,7 @@ Mutable Data 3: { "test": [{ "offset": 0, + "size": , "specimen": "" }], "write": [{ diff --git a/src/allmydata/interfaces.py b/src/allmydata/interfaces.py index dcd442624..0283b443a 100644 --- a/src/allmydata/interfaces.py +++ b/src/allmydata/interfaces.py @@ -91,8 +91,9 @@ class RIBucketReader(RemoteInterface): TestVector = ListOf(TupleOf(Offset, ReadSize, bytes, bytes)) # elements are (offset, length, operator, specimen) -# operator must be b"eq", you should use length==len(specimen). -# (These are only used for wire compatibility with old versions). +# operator must be b"eq", typically length==len(specimen), but one can ensure +# writes don't happen to empty shares by setting length to 1 and specimen to +# b"". The operator is still used for wire compatibility with old versions. DataVector = ListOf(TupleOf(Offset, ShareData)) # (offset, data). This limits us to 30 writes of 1MiB each per call TestAndWriteVectorsForShares = DictOf(int, @@ -353,7 +354,7 @@ class IStorageServer(Interface): While the interface mostly matches, test vectors are simplified. Instead of a tuple ``(start, offset, operator, data)`` they are just - ``(start, data)`` with operator implicitly being ``b"eq"``. + ``(start, data_length, data)`` with operator implicitly being ``b"eq"``. """ def advise_corrupt_share( diff --git a/src/allmydata/mutable/layout.py b/src/allmydata/mutable/layout.py index 50b60e30c..8bb2f3083 100644 --- a/src/allmydata/mutable/layout.py +++ b/src/allmydata/mutable/layout.py @@ -309,7 +309,7 @@ class SDMFSlotWriteProxy(object): salt) else: checkstring = checkstring_or_seqnum - self._testvs = [(0, checkstring)] + self._testvs = [(0, len(checkstring), checkstring)] def get_checkstring(self): @@ -318,7 +318,7 @@ class SDMFSlotWriteProxy(object): server. """ if self._testvs: - return self._testvs[0][1] + return self._testvs[0][2] return b"" @@ -548,9 +548,9 @@ class SDMFSlotWriteProxy(object): if not self._testvs: # Our caller has not provided us with another checkstring # yet, so we assume that we are writing a new share, and set - # a test vector that will allow a new share to be written. + # a test vector that will only allow a new share to be written. self._testvs = [] - self._testvs.append(tuple([0, b""])) + self._testvs.append(tuple([0, 1, b""])) tw_vectors = {} tw_vectors[self.shnum] = (self._testvs, datavs, None) @@ -889,7 +889,7 @@ class MDMFSlotWriteProxy(object): self._testvs = [] else: self._testvs = [] - self._testvs.append((0, checkstring)) + self._testvs.append((0, len(checkstring), checkstring)) def __repr__(self): @@ -1161,8 +1161,10 @@ class MDMFSlotWriteProxy(object): """I write the data vectors in datavs to the remote slot.""" tw_vectors = {} if not self._testvs: + # Make sure we will only successfully write if the share didn't + # previously exist. self._testvs = [] - self._testvs.append(tuple([0, b""])) + self._testvs.append(tuple([0, 1, b""])) if not self._written: # Write a new checkstring to the share when we write it, so # that we have something to check later. @@ -1170,7 +1172,7 @@ class MDMFSlotWriteProxy(object): datavs.append((0, new_checkstring)) def _first_write(): self._written = True - self._testvs = [(0, new_checkstring)] + self._testvs = [(0, len(new_checkstring), new_checkstring)] on_success = _first_write tw_vectors[self.shnum] = (self._testvs, datavs, None) d = self._storage_server.slot_testv_and_readv_and_writev( diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index ec877779d..ac6c107d5 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -997,7 +997,7 @@ class _StorageServer(object): # Match the wire protocol, which requires 4-tuples for test vectors. wire_format_tw_vectors = { key: ( - [(start, len(data), b"eq", data) for (start, data) in value[0]], + [(start, length, b"eq", data) for (start, length, data) in value[0]], value[1], value[2], ) for (key, value) in tw_vectors.items() From 5825b8bd42328618dfdc9f7a2aa02dd761fe2887 Mon Sep 17 00:00:00 2001 From: Florian Sesser Date: Thu, 16 Sep 2021 15:58:04 +0000 Subject: [PATCH 202/269] OpenMetrics: rework test suite with exarkun --- src/allmydata/test/test_openmetrics.py | 90 ++++++++++++++++++++------ 1 file changed, 72 insertions(+), 18 deletions(-) diff --git a/src/allmydata/test/test_openmetrics.py b/src/allmydata/test/test_openmetrics.py index 7d9bd6429..a0822d594 100644 --- a/src/allmydata/test/test_openmetrics.py +++ b/src/allmydata/test/test_openmetrics.py @@ -1,7 +1,25 @@ -import mock from prometheus_client.openmetrics import parser -from twisted.trial import unittest + +from treq.testing import RequestTraversalAgent + +from twisted.web.http import OK +from twisted.web.client import readBody +from twisted.web.resource import Resource + +from testtools.twistedsupport import succeeded +from testtools.matchers import ( + Always, + AfterPreprocessing, + Equals, + MatchesAll, + MatchesStructure, + MatchesPredicate, + ) +from testtools.content import text_content + from allmydata.web.status import Statistics +from allmydata.test.common import SyncTestCase + class FakeStatsProvider(object): def get_stats(self): @@ -97,31 +115,67 @@ class FakeStatsProvider(object): } return stats -class FakeStats(Statistics): - def __init__(self): - self._provider = FakeStatsProvider() +class HackItResource(Resource): + def getChildWithDefault(self, path, request): + request.fields = None + return Resource.getChildWithDefault(self, path, request) -class OpenMetrics(unittest.TestCase): + +class OpenMetrics(SyncTestCase): def test_spec_compliance(self): """ Does our output adhere to the `OpenMetrics ` spec? https://github.com/OpenObservability/OpenMetrics/ https://prometheus.io/docs/instrumenting/exposition_formats/ """ - req = mock.Mock() - stats = FakeStats() - metrics = Statistics.render_OPENMETRICS(stats, req) + root = HackItResource() + root.putChild(b"", Statistics(FakeStatsProvider())) + rta = RequestTraversalAgent(root) + d = rta.request(b"GET", b"http://localhost/?t=openmetrics") + self.assertThat(d, succeeded(matches_stats(self))) - # "The content type MUST be..." - req.setHeader.assert_called_with( - "content-type", "application/openmetrics-text; version=1.0.0; charset=utf-8" +def matches_stats(testcase): + def add_detail(testcase): + def predicate(body): + testcase.addDetail("body", text_content(body)) + return True + return predicate + + return MatchesAll( + MatchesStructure( + code=Equals(OK), + # "The content type MUST be..." + headers=has_header("content-type", "application/openmetrics-text; version=1.0.0; charset=utf-8"), + ), + AfterPreprocessing( + readBodyText, + succeeded(MatchesAll( + MatchesPredicate(add_detail(testcase), "%s dummy"), + parses_as_openmetrics(), + )) ) + ) - # The parser throws if it does not like its input. - # Wrapped in a list() to drain the generator. - families = list(parser.text_string_to_metric_families(metrics)) +def readBodyText(response): + d = readBody(response) + d.addCallback(lambda body: body.decode("utf-8")) + return d + +def has_header(name, value): + return AfterPreprocessing( + lambda headers: headers.getRawHeaders(name), + Equals([value]), + ) + +def parses_as_openmetrics(): + # The parser throws if it does not like its input. + # Wrapped in a list() to drain the generator. + return AfterPreprocessing( + lambda body: list(parser.text_string_to_metric_families(body)), + AfterPreprocessing( + lambda families: families[-1].name, + Equals(u"tahoe_stats_storage_server_total_bucket_count"), + ), + ) - # Has the parser parsed our data? - # Just check the last item. - self.assertEqual(families[-1].name, u"tahoe_stats_storage_server_total_bucket_count") From a2378d0e704add3741aa269adc37ac3f781ac36a Mon Sep 17 00:00:00 2001 From: Florian Sesser Date: Fri, 17 Sep 2021 12:04:12 +0000 Subject: [PATCH 203/269] OpenMetrics test suite: Make CI happy: No old style objects --- src/allmydata/test/test_openmetrics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/test_openmetrics.py b/src/allmydata/test/test_openmetrics.py index a0822d594..43704c4a3 100644 --- a/src/allmydata/test/test_openmetrics.py +++ b/src/allmydata/test/test_openmetrics.py @@ -115,7 +115,7 @@ class FakeStatsProvider(object): } return stats -class HackItResource(Resource): +class HackItResource(Resource, object): def getChildWithDefault(self, path, request): request.fields = None return Resource.getChildWithDefault(self, path, request) From fb0335cc17c709d5663f7ea2c31ccb98b6d6ddf1 Mon Sep 17 00:00:00 2001 From: Florian Sesser Date: Fri, 17 Sep 2021 12:16:37 +0000 Subject: [PATCH 204/269] OpenMetrics test suite: More clean up The Linter complains: > 'testtools.matchers.Always' imported but unused --- src/allmydata/test/test_openmetrics.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/allmydata/test/test_openmetrics.py b/src/allmydata/test/test_openmetrics.py index 43704c4a3..91692af33 100644 --- a/src/allmydata/test/test_openmetrics.py +++ b/src/allmydata/test/test_openmetrics.py @@ -8,7 +8,6 @@ from twisted.web.resource import Resource from testtools.twistedsupport import succeeded from testtools.matchers import ( - Always, AfterPreprocessing, Equals, MatchesAll, From e5e0d71ef518c73e3fe2a0bc860e1c7428ba91a9 Mon Sep 17 00:00:00 2001 From: Florian Sesser Date: Fri, 17 Sep 2021 13:20:59 +0000 Subject: [PATCH 205/269] OpenMetrics test suite: More clean ups trailing whitespace --- src/allmydata/test/test_openmetrics.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/allmydata/test/test_openmetrics.py b/src/allmydata/test/test_openmetrics.py index 91692af33..9677de4a9 100644 --- a/src/allmydata/test/test_openmetrics.py +++ b/src/allmydata/test/test_openmetrics.py @@ -3,7 +3,7 @@ from prometheus_client.openmetrics import parser from treq.testing import RequestTraversalAgent from twisted.web.http import OK -from twisted.web.client import readBody +from twisted.web.client import readBody from twisted.web.resource import Resource from testtools.twistedsupport import succeeded @@ -128,7 +128,7 @@ class OpenMetrics(SyncTestCase): https://prometheus.io/docs/instrumenting/exposition_formats/ """ root = HackItResource() - root.putChild(b"", Statistics(FakeStatsProvider())) + root.putChild(b"", Statistics(FakeStatsProvider())) rta = RequestTraversalAgent(root) d = rta.request(b"GET", b"http://localhost/?t=openmetrics") self.assertThat(d, succeeded(matches_stats(self))) @@ -168,11 +168,11 @@ def has_header(name, value): def parses_as_openmetrics(): # The parser throws if it does not like its input. - # Wrapped in a list() to drain the generator. + # Wrapped in a list() to drain the generator. return AfterPreprocessing( lambda body: list(parser.text_string_to_metric_families(body)), AfterPreprocessing( - lambda families: families[-1].name, + lambda families: families[-1].name, Equals(u"tahoe_stats_storage_server_total_bucket_count"), ), ) From 460b74e171c97aa14999fbb8fa41469d9d6e1da7 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 21 Sep 2021 13:29:24 -0400 Subject: [PATCH 206/269] Punt overlapping writes for now. --- src/allmydata/test/test_istorageserver.py | 27 +++++++++-------------- 1 file changed, 10 insertions(+), 17 deletions(-) diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index 1c1383a2d..328000489 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -211,11 +211,8 @@ class IStorageServerImmutableAPIsTestsMixin(object): """ Shares that are fully written to can be read. - 1. The result is not affected by the order in which writes - happened, only by their offsets. - - 2. When overlapping writes happen, the resulting read returns the - latest written value. + The result is not affected by the order in which writes + happened, only by their offsets. """ storage_index, renew_secret, cancel_secret = ( new_storage_index(), @@ -241,15 +238,8 @@ class IStorageServerImmutableAPIsTestsMixin(object): yield allocated[2].callRemote("write", 0, b"3" * 512) yield allocated[2].callRemote("close") - # Bucket 3 has an overlapping write. - yield allocated[3].callRemote("write", 0, b"5" * 20) - # The second write will overwrite the first. - yield allocated[3].callRemote("write", 0, b"6" * 24) - yield allocated[3].callRemote("write", 24, b"7" * 1000) - yield allocated[3].callRemote("close") - buckets = yield self.storage_server.get_buckets(storage_index) - self.assertEqual(set(buckets.keys()), {1, 2, 3}) + self.assertEqual(set(buckets.keys()), {1, 2}) self.assertEqual( (yield buckets[1].callRemote("read", 0, 1024)), b"1" * 512 + b"2" * 512 @@ -257,10 +247,13 @@ class IStorageServerImmutableAPIsTestsMixin(object): self.assertEqual( (yield buckets[2].callRemote("read", 0, 1024)), b"3" * 512 + b"4" * 512 ) - self.assertEqual( - (yield buckets[3].callRemote("read", 0, 1024)), - b"6" * 24 + b"7" * 1000, - ) + + @skipIf(True, "https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3801") + def test_overlapping_writes(self): + """ + The policy for overlapping writes is TBD: + https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3801 + """ class _FoolscapMixin(SystemTestMixin): From c80016d2ecd0e6544d6f61ebe72e605c59767c52 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 22 Sep 2021 15:51:26 -0400 Subject: [PATCH 207/269] convert tab to spaces --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index f40f6f3ba..610570be5 100644 --- a/tox.ini +++ b/tox.ini @@ -74,7 +74,7 @@ commands = tahoe --version - python -c "import sys; print('sys.stdout.encoding:', sys.stdout.encoding)" + python -c "import sys; print('sys.stdout.encoding:', sys.stdout.encoding)" # Run tests with -b to catch bugs like `"%s" % (some_bytes,)`. -b makes # Python emit BytesWarnings, and warnings configuration in From 198de5c6482b0b7502c0753397713112914d7145 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 22 Sep 2021 15:53:56 -0400 Subject: [PATCH 208/269] Hoist the sys.stdout import to top level --- src/allmydata/scripts/tahoe_status.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/allmydata/scripts/tahoe_status.py b/src/allmydata/scripts/tahoe_status.py index e73958642..250bfdea3 100644 --- a/src/allmydata/scripts/tahoe_status.py +++ b/src/allmydata/scripts/tahoe_status.py @@ -11,6 +11,7 @@ if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 import os +from sys import stdout as _sys_stdout from urllib.parse import urlencode import json @@ -24,13 +25,17 @@ from allmydata.util.encodingutil import argv_to_abspath _print = print def print(*args, **kwargs): """ - Builtin ``print``-alike that will even write unicode not encodeable using - the specified output file's encoding. + Builtin ``print``-alike that will even write unicode which cannot be + encoded using the specified output file's encoding. + + This differs from the builtin print in that it will use the "replace" + encoding error handler and then write the result whereas builtin print + uses the "strict" encoding error handler. """ from past.builtins import unicode out = kwargs.pop("file", None) if out is None: - from sys import stdout as out + out = _sys_stdout encoding = out.encoding or "ascii" def ensafe(o): if isinstance(o, unicode): From 5e26f25b3714167a628e93ae76a529b074fb4c69 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 23 Sep 2021 07:41:43 -0400 Subject: [PATCH 209/269] It's ported to Python 3! --- src/allmydata/test/test_openmetrics.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/allmydata/test/test_openmetrics.py b/src/allmydata/test/test_openmetrics.py index 9677de4a9..350d6abbd 100644 --- a/src/allmydata/test/test_openmetrics.py +++ b/src/allmydata/test/test_openmetrics.py @@ -1,3 +1,18 @@ +""" +Tests for ``/statistics?t=openmetrics``. + +Ported to Python 3. +""" + +from __future__ import print_function +from __future__ import absolute_import +from __future__ import division +from __future__ import unicode_literals + +from future.utils import PY2 +if PY2: + from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + from prometheus_client.openmetrics import parser from treq.testing import RequestTraversalAgent From f8c07bfd11edfee9b62b8bb13cbe46f643267322 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 23 Sep 2021 07:42:59 -0400 Subject: [PATCH 210/269] add some docstrings --- src/allmydata/test/test_openmetrics.py | 37 ++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/src/allmydata/test/test_openmetrics.py b/src/allmydata/test/test_openmetrics.py index 350d6abbd..34dcd266b 100644 --- a/src/allmydata/test/test_openmetrics.py +++ b/src/allmydata/test/test_openmetrics.py @@ -36,6 +36,10 @@ from allmydata.test.common import SyncTestCase class FakeStatsProvider(object): + """ + A stats provider that hands backed a canned collection of performance + statistics. + """ def get_stats(self): # Parsed into a dict from a running tahoe's /statistics?t=json stats = { @@ -130,12 +134,21 @@ class FakeStatsProvider(object): return stats class HackItResource(Resource, object): + """ + A bridge between ``RequestTraversalAgent`` and ``MultiFormatResource`` + (used by ``Statistics``). ``MultiFormatResource`` expects the request + object to have a ``fields`` attribute but Twisted's ``IRequest`` has no + such attribute. Create it here. + """ def getChildWithDefault(self, path, request): request.fields = None return Resource.getChildWithDefault(self, path, request) class OpenMetrics(SyncTestCase): + """ + Tests for ``/statistics?t=openmetrics``. + """ def test_spec_compliance(self): """ Does our output adhere to the `OpenMetrics ` spec? @@ -171,17 +184,41 @@ def matches_stats(testcase): ) def readBodyText(response): + """ + Read the response body and decode it using UTF-8. + + :param twisted.web.iweb.IResponse response: The response from which to + read the body. + + :return: A ``Deferred`` that fires with the ``str`` body. + """ d = readBody(response) d.addCallback(lambda body: body.decode("utf-8")) return d def has_header(name, value): + """ + Create a matcher that matches a response object that includes the given + name / value pair. + + :param str name: The name of the item in the HTTP header to match. + :param str value: The value of the item in the HTTP header to match by equality. + + :return: A matcher. + """ return AfterPreprocessing( lambda headers: headers.getRawHeaders(name), Equals([value]), ) def parses_as_openmetrics(): + """ + Create a matcher that matches a ``str`` string that can be parsed as an + OpenMetrics response and includes a certain well-known value expected by + the tests. + + :return: A matcher. + """ # The parser throws if it does not like its input. # Wrapped in a list() to drain the generator. return AfterPreprocessing( From 4d8164773c404373f4aeea37f2969680fa02c265 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 23 Sep 2021 07:43:18 -0400 Subject: [PATCH 211/269] factor helper function out to top-level --- src/allmydata/test/test_openmetrics.py | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/src/allmydata/test/test_openmetrics.py b/src/allmydata/test/test_openmetrics.py index 34dcd266b..a92098c88 100644 --- a/src/allmydata/test/test_openmetrics.py +++ b/src/allmydata/test/test_openmetrics.py @@ -162,11 +162,6 @@ class OpenMetrics(SyncTestCase): self.assertThat(d, succeeded(matches_stats(self))) def matches_stats(testcase): - def add_detail(testcase): - def predicate(body): - testcase.addDetail("body", text_content(body)) - return True - return predicate return MatchesAll( MatchesStructure( @@ -177,12 +172,26 @@ def matches_stats(testcase): AfterPreprocessing( readBodyText, succeeded(MatchesAll( - MatchesPredicate(add_detail(testcase), "%s dummy"), + MatchesPredicate(add_detail(testcase, u"response body"), u"%s dummy"), parses_as_openmetrics(), )) ) ) +def add_detail(testcase, name): + """ + Create a matcher that always matches and as a side-effect adds the matched + value as detail to the testcase. + + :param testtools.TestCase testcase: The case to which to add the detail. + + :return: A matcher. + """ + def predicate(value): + testcase.addDetail(name, text_content(value)) + return True + return predicate + def readBodyText(response): """ Read the response body and decode it using UTF-8. From cbb96bd57a2fa3fa5a0a11662ea4890a5ad9bc41 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 23 Sep 2021 07:43:33 -0400 Subject: [PATCH 212/269] one more docstring --- src/allmydata/test/test_openmetrics.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/allmydata/test/test_openmetrics.py b/src/allmydata/test/test_openmetrics.py index a92098c88..385bf32a8 100644 --- a/src/allmydata/test/test_openmetrics.py +++ b/src/allmydata/test/test_openmetrics.py @@ -162,7 +162,20 @@ class OpenMetrics(SyncTestCase): self.assertThat(d, succeeded(matches_stats(self))) def matches_stats(testcase): + """ + Create a matcher that matches a response that confirms to the OpenMetrics + specification. + * The ``Content-Type`` is **application/openmetrics-text; version=1.0.0; charset=utf-8**. + * The status is **OK**. + * The body can be parsed by an OpenMetrics parser. + * The metric families in the body are grouped and sorted. + * At least one of the expected families appears in the body. + + :param testtools.TestCase testcase: The case to which to add detail about the matching process. + + :return: A matcher. + """ return MatchesAll( MatchesStructure( code=Equals(OK), From f66a8ab1369f197163832a940c25efdb44f3dcde Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 23 Sep 2021 07:43:37 -0400 Subject: [PATCH 213/269] formatting and explicit unicode string literals --- src/allmydata/test/test_openmetrics.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/allmydata/test/test_openmetrics.py b/src/allmydata/test/test_openmetrics.py index 385bf32a8..d74958ffd 100644 --- a/src/allmydata/test/test_openmetrics.py +++ b/src/allmydata/test/test_openmetrics.py @@ -180,7 +180,10 @@ def matches_stats(testcase): MatchesStructure( code=Equals(OK), # "The content type MUST be..." - headers=has_header("content-type", "application/openmetrics-text; version=1.0.0; charset=utf-8"), + headers=has_header( + u"content-type", + u"application/openmetrics-text; version=1.0.0; charset=utf-8", + ), ), AfterPreprocessing( readBodyText, From 4b6d00221e289a94fa9603dc0d1050605c0411ab Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 23 Sep 2021 07:48:19 -0400 Subject: [PATCH 214/269] protect this crazy line from black --- src/allmydata/test/test_openmetrics.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/allmydata/test/test_openmetrics.py b/src/allmydata/test/test_openmetrics.py index d74958ffd..71433c11a 100644 --- a/src/allmydata/test/test_openmetrics.py +++ b/src/allmydata/test/test_openmetrics.py @@ -10,8 +10,11 @@ from __future__ import division from __future__ import unicode_literals from future.utils import PY2 + if PY2: + # fmt: off from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 + # fmt: on from prometheus_client.openmetrics import parser From 2f60ab300ba724bd67c0ca0825af089eecf13e36 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 23 Sep 2021 07:48:27 -0400 Subject: [PATCH 215/269] black formatting --- src/allmydata/test/test_openmetrics.py | 211 +++++++++++++------------ 1 file changed, 112 insertions(+), 99 deletions(-) diff --git a/src/allmydata/test/test_openmetrics.py b/src/allmydata/test/test_openmetrics.py index 71433c11a..66cbc7dec 100644 --- a/src/allmydata/test/test_openmetrics.py +++ b/src/allmydata/test/test_openmetrics.py @@ -31,7 +31,7 @@ from testtools.matchers import ( MatchesAll, MatchesStructure, MatchesPredicate, - ) +) from testtools.content import text_content from allmydata.web.status import Statistics @@ -43,99 +43,103 @@ class FakeStatsProvider(object): A stats provider that hands backed a canned collection of performance statistics. """ + def get_stats(self): # Parsed into a dict from a running tahoe's /statistics?t=json stats = { - 'stats': { - 'storage_server.latencies.get.99_9_percentile': None, - 'storage_server.latencies.close.10_0_percentile': 0.00021910667419433594, - 'storage_server.latencies.read.01_0_percentile': 2.8848648071289062e-05, - 'storage_server.latencies.writev.99_9_percentile': None, - 'storage_server.latencies.read.99_9_percentile': None, - 'storage_server.latencies.allocate.99_0_percentile': 0.000988006591796875, - 'storage_server.latencies.writev.mean': 0.00045332245070571654, - 'storage_server.latencies.close.99_9_percentile': None, - 'cpu_monitor.15min_avg': 0.00017592000079223033, - 'storage_server.disk_free_for_root': 103289454592, - 'storage_server.latencies.get.99_0_percentile': 0.000347137451171875, - 'storage_server.latencies.get.mean': 0.00021158285060171353, - 'storage_server.latencies.read.90_0_percentile': 8.893013000488281e-05, - 'storage_server.latencies.write.01_0_percentile': 3.600120544433594e-05, - 'storage_server.latencies.write.99_9_percentile': 0.00017690658569335938, - 'storage_server.latencies.close.90_0_percentile': 0.00033211708068847656, - 'storage_server.disk_total': 103497859072, - 'storage_server.latencies.close.95_0_percentile': 0.0003509521484375, - 'storage_server.latencies.readv.samplesize': 1000, - 'storage_server.disk_free_for_nonroot': 103289454592, - 'storage_server.latencies.close.mean': 0.0002715024480059103, - 'storage_server.latencies.writev.95_0_percentile': 0.0007410049438476562, - 'storage_server.latencies.readv.90_0_percentile': 0.0003781318664550781, - 'storage_server.latencies.readv.99_0_percentile': 0.0004050731658935547, - 'storage_server.latencies.allocate.mean': 0.0007128627429454784, - 'storage_server.latencies.close.samplesize': 326, - 'storage_server.latencies.get.50_0_percentile': 0.0001819133758544922, - 'storage_server.latencies.write.50_0_percentile': 4.482269287109375e-05, - 'storage_server.latencies.readv.01_0_percentile': 0.0002970695495605469, - 'storage_server.latencies.get.10_0_percentile': 0.00015687942504882812, - 'storage_server.latencies.allocate.90_0_percentile': 0.0008189678192138672, - 'storage_server.latencies.get.samplesize': 472, - 'storage_server.total_bucket_count': 393, - 'storage_server.latencies.read.mean': 5.936201880959903e-05, - 'storage_server.latencies.allocate.01_0_percentile': 0.0004208087921142578, - 'storage_server.latencies.allocate.99_9_percentile': None, - 'storage_server.latencies.readv.mean': 0.00034061360359191893, - 'storage_server.disk_used': 208404480, - 'storage_server.latencies.allocate.50_0_percentile': 0.0007410049438476562, - 'storage_server.latencies.read.99_0_percentile': 0.00011992454528808594, - 'node.uptime': 3805759.8545179367, - 'storage_server.latencies.writev.10_0_percentile': 0.00035190582275390625, - 'storage_server.latencies.writev.90_0_percentile': 0.0006821155548095703, - 'storage_server.latencies.close.01_0_percentile': 0.00021505355834960938, - 'storage_server.latencies.close.50_0_percentile': 0.0002579689025878906, - 'cpu_monitor.1min_avg': 0.0002130000000003444, - 'storage_server.latencies.writev.50_0_percentile': 0.0004138946533203125, - 'storage_server.latencies.read.95_0_percentile': 9.107589721679688e-05, - 'storage_server.latencies.readv.95_0_percentile': 0.0003859996795654297, - 'storage_server.latencies.write.10_0_percentile': 3.719329833984375e-05, - 'storage_server.accepting_immutable_shares': 1, - 'storage_server.latencies.writev.samplesize': 309, - 'storage_server.latencies.get.95_0_percentile': 0.0003190040588378906, - 'storage_server.latencies.readv.10_0_percentile': 0.00032210350036621094, - 'storage_server.latencies.get.90_0_percentile': 0.0002999305725097656, - 'storage_server.latencies.get.01_0_percentile': 0.0001239776611328125, - 'cpu_monitor.total': 641.4941180000001, - 'storage_server.latencies.write.samplesize': 1000, - 'storage_server.latencies.write.95_0_percentile': 9.489059448242188e-05, - 'storage_server.latencies.read.50_0_percentile': 6.890296936035156e-05, - 'storage_server.latencies.writev.01_0_percentile': 0.00033211708068847656, - 'storage_server.latencies.read.10_0_percentile': 3.0994415283203125e-05, - 'storage_server.latencies.allocate.10_0_percentile': 0.0004949569702148438, - 'storage_server.reserved_space': 0, - 'storage_server.disk_avail': 103289454592, - 'storage_server.latencies.write.99_0_percentile': 0.00011301040649414062, - 'storage_server.latencies.write.90_0_percentile': 9.083747863769531e-05, - 'cpu_monitor.5min_avg': 0.0002370666691157502, - 'storage_server.latencies.write.mean': 5.8008909225463864e-05, - 'storage_server.latencies.readv.50_0_percentile': 0.00033020973205566406, - 'storage_server.latencies.close.99_0_percentile': 0.0004038810729980469, - 'storage_server.allocated': 0, - 'storage_server.latencies.writev.99_0_percentile': 0.0007710456848144531, - 'storage_server.latencies.readv.99_9_percentile': 0.0004780292510986328, - 'storage_server.latencies.read.samplesize': 170, - 'storage_server.latencies.allocate.samplesize': 406, - 'storage_server.latencies.allocate.95_0_percentile': 0.0008411407470703125}, - 'counters': { - 'storage_server.writev': 309, - 'storage_server.bytes_added': 197836146, - 'storage_server.close': 326, - 'storage_server.readv': 14299, - 'storage_server.allocate': 406, - 'storage_server.read': 170, - 'storage_server.write': 3775, - 'storage_server.get': 472} - } + "stats": { + "storage_server.latencies.get.99_9_percentile": None, + "storage_server.latencies.close.10_0_percentile": 0.00021910667419433594, + "storage_server.latencies.read.01_0_percentile": 2.8848648071289062e-05, + "storage_server.latencies.writev.99_9_percentile": None, + "storage_server.latencies.read.99_9_percentile": None, + "storage_server.latencies.allocate.99_0_percentile": 0.000988006591796875, + "storage_server.latencies.writev.mean": 0.00045332245070571654, + "storage_server.latencies.close.99_9_percentile": None, + "cpu_monitor.15min_avg": 0.00017592000079223033, + "storage_server.disk_free_for_root": 103289454592, + "storage_server.latencies.get.99_0_percentile": 0.000347137451171875, + "storage_server.latencies.get.mean": 0.00021158285060171353, + "storage_server.latencies.read.90_0_percentile": 8.893013000488281e-05, + "storage_server.latencies.write.01_0_percentile": 3.600120544433594e-05, + "storage_server.latencies.write.99_9_percentile": 0.00017690658569335938, + "storage_server.latencies.close.90_0_percentile": 0.00033211708068847656, + "storage_server.disk_total": 103497859072, + "storage_server.latencies.close.95_0_percentile": 0.0003509521484375, + "storage_server.latencies.readv.samplesize": 1000, + "storage_server.disk_free_for_nonroot": 103289454592, + "storage_server.latencies.close.mean": 0.0002715024480059103, + "storage_server.latencies.writev.95_0_percentile": 0.0007410049438476562, + "storage_server.latencies.readv.90_0_percentile": 0.0003781318664550781, + "storage_server.latencies.readv.99_0_percentile": 0.0004050731658935547, + "storage_server.latencies.allocate.mean": 0.0007128627429454784, + "storage_server.latencies.close.samplesize": 326, + "storage_server.latencies.get.50_0_percentile": 0.0001819133758544922, + "storage_server.latencies.write.50_0_percentile": 4.482269287109375e-05, + "storage_server.latencies.readv.01_0_percentile": 0.0002970695495605469, + "storage_server.latencies.get.10_0_percentile": 0.00015687942504882812, + "storage_server.latencies.allocate.90_0_percentile": 0.0008189678192138672, + "storage_server.latencies.get.samplesize": 472, + "storage_server.total_bucket_count": 393, + "storage_server.latencies.read.mean": 5.936201880959903e-05, + "storage_server.latencies.allocate.01_0_percentile": 0.0004208087921142578, + "storage_server.latencies.allocate.99_9_percentile": None, + "storage_server.latencies.readv.mean": 0.00034061360359191893, + "storage_server.disk_used": 208404480, + "storage_server.latencies.allocate.50_0_percentile": 0.0007410049438476562, + "storage_server.latencies.read.99_0_percentile": 0.00011992454528808594, + "node.uptime": 3805759.8545179367, + "storage_server.latencies.writev.10_0_percentile": 0.00035190582275390625, + "storage_server.latencies.writev.90_0_percentile": 0.0006821155548095703, + "storage_server.latencies.close.01_0_percentile": 0.00021505355834960938, + "storage_server.latencies.close.50_0_percentile": 0.0002579689025878906, + "cpu_monitor.1min_avg": 0.0002130000000003444, + "storage_server.latencies.writev.50_0_percentile": 0.0004138946533203125, + "storage_server.latencies.read.95_0_percentile": 9.107589721679688e-05, + "storage_server.latencies.readv.95_0_percentile": 0.0003859996795654297, + "storage_server.latencies.write.10_0_percentile": 3.719329833984375e-05, + "storage_server.accepting_immutable_shares": 1, + "storage_server.latencies.writev.samplesize": 309, + "storage_server.latencies.get.95_0_percentile": 0.0003190040588378906, + "storage_server.latencies.readv.10_0_percentile": 0.00032210350036621094, + "storage_server.latencies.get.90_0_percentile": 0.0002999305725097656, + "storage_server.latencies.get.01_0_percentile": 0.0001239776611328125, + "cpu_monitor.total": 641.4941180000001, + "storage_server.latencies.write.samplesize": 1000, + "storage_server.latencies.write.95_0_percentile": 9.489059448242188e-05, + "storage_server.latencies.read.50_0_percentile": 6.890296936035156e-05, + "storage_server.latencies.writev.01_0_percentile": 0.00033211708068847656, + "storage_server.latencies.read.10_0_percentile": 3.0994415283203125e-05, + "storage_server.latencies.allocate.10_0_percentile": 0.0004949569702148438, + "storage_server.reserved_space": 0, + "storage_server.disk_avail": 103289454592, + "storage_server.latencies.write.99_0_percentile": 0.00011301040649414062, + "storage_server.latencies.write.90_0_percentile": 9.083747863769531e-05, + "cpu_monitor.5min_avg": 0.0002370666691157502, + "storage_server.latencies.write.mean": 5.8008909225463864e-05, + "storage_server.latencies.readv.50_0_percentile": 0.00033020973205566406, + "storage_server.latencies.close.99_0_percentile": 0.0004038810729980469, + "storage_server.allocated": 0, + "storage_server.latencies.writev.99_0_percentile": 0.0007710456848144531, + "storage_server.latencies.readv.99_9_percentile": 0.0004780292510986328, + "storage_server.latencies.read.samplesize": 170, + "storage_server.latencies.allocate.samplesize": 406, + "storage_server.latencies.allocate.95_0_percentile": 0.0008411407470703125, + }, + "counters": { + "storage_server.writev": 309, + "storage_server.bytes_added": 197836146, + "storage_server.close": 326, + "storage_server.readv": 14299, + "storage_server.allocate": 406, + "storage_server.read": 170, + "storage_server.write": 3775, + "storage_server.get": 472, + }, + } return stats + class HackItResource(Resource, object): """ A bridge between ``RequestTraversalAgent`` and ``MultiFormatResource`` @@ -143,6 +147,7 @@ class HackItResource(Resource, object): object to have a ``fields`` attribute but Twisted's ``IRequest`` has no such attribute. Create it here. """ + def getChildWithDefault(self, path, request): request.fields = None return Resource.getChildWithDefault(self, path, request) @@ -152,6 +157,7 @@ class OpenMetrics(SyncTestCase): """ Tests for ``/statistics?t=openmetrics``. """ + def test_spec_compliance(self): """ Does our output adhere to the `OpenMetrics ` spec? @@ -164,6 +170,7 @@ class OpenMetrics(SyncTestCase): d = rta.request(b"GET", b"http://localhost/?t=openmetrics") self.assertThat(d, succeeded(matches_stats(self))) + def matches_stats(testcase): """ Create a matcher that matches a response that confirms to the OpenMetrics @@ -184,19 +191,22 @@ def matches_stats(testcase): code=Equals(OK), # "The content type MUST be..." headers=has_header( - u"content-type", - u"application/openmetrics-text; version=1.0.0; charset=utf-8", + "content-type", + "application/openmetrics-text; version=1.0.0; charset=utf-8", ), ), AfterPreprocessing( readBodyText, - succeeded(MatchesAll( - MatchesPredicate(add_detail(testcase, u"response body"), u"%s dummy"), - parses_as_openmetrics(), - )) - ) + succeeded( + MatchesAll( + MatchesPredicate(add_detail(testcase, "response body"), "%s dummy"), + parses_as_openmetrics(), + ) + ), + ), ) + def add_detail(testcase, name): """ Create a matcher that always matches and as a side-effect adds the matched @@ -206,11 +216,14 @@ def add_detail(testcase, name): :return: A matcher. """ + def predicate(value): testcase.addDetail(name, text_content(value)) return True + return predicate + def readBodyText(response): """ Read the response body and decode it using UTF-8. @@ -224,6 +237,7 @@ def readBodyText(response): d.addCallback(lambda body: body.decode("utf-8")) return d + def has_header(name, value): """ Create a matcher that matches a response object that includes the given @@ -239,6 +253,7 @@ def has_header(name, value): Equals([value]), ) + def parses_as_openmetrics(): """ Create a matcher that matches a ``str`` string that can be parsed as an @@ -253,8 +268,6 @@ def parses_as_openmetrics(): lambda body: list(parser.text_string_to_metric_families(body)), AfterPreprocessing( lambda families: families[-1].name, - Equals(u"tahoe_stats_storage_server_total_bucket_count"), + Equals("tahoe_stats_storage_server_total_bucket_count"), ), ) - - From 7183d53c23e3126032a59b5617259af240ead552 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 23 Sep 2021 07:58:02 -0400 Subject: [PATCH 216/269] put test dependency in the setuptools test extra --- setup.py | 2 ++ tox.ini | 2 -- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 3433e93f4..27407a403 100644 --- a/setup.py +++ b/setup.py @@ -404,6 +404,8 @@ setup(name="tahoe-lafs", # also set in __init__.py "tenacity", "paramiko", "pytest-timeout", + # Does our OpenMetrics endpoint adhere to the spec: + "prometheus-client == 0.11.0", ] + tor_requires + i2p_requires, "tor": tor_requires, "i2p": i2p_requires, diff --git a/tox.ini b/tox.ini index 0e8e58ea6..9b0f71038 100644 --- a/tox.ini +++ b/tox.ini @@ -52,8 +52,6 @@ deps = certifi # VCS hooks support py36,!coverage: pre-commit - # Does our OpenMetrics endpoint adhere to the spec: - prometheus-client==0.11.0 # We add usedevelop=False because testing against a true installation gives # more useful results. From ec6dfb8297f7903911f4ecfdd2278e720709af09 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 23 Sep 2021 14:20:34 -0400 Subject: [PATCH 217/269] Re-enable test. --- src/allmydata/test/test_istorageserver.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index 328000489..a63189204 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -248,12 +248,11 @@ class IStorageServerImmutableAPIsTestsMixin(object): (yield buckets[2].callRemote("read", 0, 1024)), b"3" * 512 + b"4" * 512 ) - @skipIf(True, "https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3801") def test_overlapping_writes(self): """ - The policy for overlapping writes is TBD: - https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3801 + Overlapping writes in immutable uploads fail with ``OverlappingWriteError``. """ + 1/0 class _FoolscapMixin(SystemTestMixin): From 1ff4e61e417a5f73f89ed46148dd3e8808a471ed Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 24 Sep 2021 10:33:26 -0400 Subject: [PATCH 218/269] Low-level tests for conflicting and non-conflicting writes. --- src/allmydata/interfaces.py | 8 +++ src/allmydata/storage/common.py | 5 +- src/allmydata/test/test_storage.py | 96 +++++++++++++++++++++++++++++- 3 files changed, 104 insertions(+), 5 deletions(-) diff --git a/src/allmydata/interfaces.py b/src/allmydata/interfaces.py index 1c64bce8a..8dffa9e7e 100644 --- a/src/allmydata/interfaces.py +++ b/src/allmydata/interfaces.py @@ -53,6 +53,14 @@ LeaseRenewSecret = Hash # used to protect lease renewal requests LeaseCancelSecret = Hash # was used to protect lease cancellation requests +class DataTooLargeError(Exception): + """The write went past the expected size of the bucket.""" + + +class ConflictingWriteError(Exception): + """Two writes happened to same immutable with different data.""" + + class RIBucketWriter(RemoteInterface): """ Objects of this kind live on the server side. """ def write(offset=Offset, data=ShareData): diff --git a/src/allmydata/storage/common.py b/src/allmydata/storage/common.py index cb6116e5b..d72bb3fbc 100644 --- a/src/allmydata/storage/common.py +++ b/src/allmydata/storage/common.py @@ -13,8 +13,9 @@ if PY2: import os.path from allmydata.util import base32 -class DataTooLargeError(Exception): - pass +# Backwards compatibility. +from allmydata.interfaces import DataTooLargeError + class UnknownMutableContainerVersionError(Exception): pass class UnknownImmutableContainerVersionError(Exception): diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index bd0ab80f3..61494eebd 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -8,7 +8,7 @@ from __future__ import division from __future__ import print_function from __future__ import unicode_literals -from future.utils import native_str, PY2, bytes_to_native_str +from future.utils import native_str, PY2, bytes_to_native_str, bchr if PY2: from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 from six import ensure_str @@ -20,12 +20,15 @@ import stat import struct import shutil import gc +from uuid import uuid4 from twisted.trial import unittest from twisted.internet import defer from twisted.internet.task import Clock +from hypothesis import given, strategies + import itertools from allmydata import interfaces from allmydata.util import fileutil, hashutil, base32 @@ -33,7 +36,7 @@ from allmydata.storage.server import StorageServer, DEFAULT_RENEWAL_TIME from allmydata.storage.shares import get_share_file from allmydata.storage.mutable import MutableShareFile from allmydata.storage.immutable import BucketWriter, BucketReader, ShareFile -from allmydata.storage.common import DataTooLargeError, storage_index_to_dir, \ +from allmydata.storage.common import storage_index_to_dir, \ UnknownMutableContainerVersionError, UnknownImmutableContainerVersionError, \ si_b2a, si_a2b from allmydata.storage.lease import LeaseInfo @@ -47,7 +50,9 @@ from allmydata.mutable.layout import MDMFSlotWriteProxy, MDMFSlotReadProxy, \ SIGNATURE_SIZE, \ VERIFICATION_KEY_SIZE, \ SHARE_HASH_CHAIN_SIZE -from allmydata.interfaces import BadWriteEnablerError +from allmydata.interfaces import ( + BadWriteEnablerError, DataTooLargeError, ConflictingWriteError, +) from allmydata.test.no_network import NoNetworkServer from allmydata.storage_client import ( _StorageServer, @@ -147,6 +152,91 @@ class Bucket(unittest.TestCase): self.failUnlessEqual(br.remote_read(25, 25), b"b"*25) self.failUnlessEqual(br.remote_read(50, 7), b"c"*7) + def test_write_past_size_errors(self): + """Writing beyond the size of the bucket throws an exception.""" + for (i, (offset, length)) in enumerate([(0, 201), (10, 191), (202, 34)]): + incoming, final = self.make_workdir( + "test_write_past_size_errors-{}".format(i) + ) + bw = BucketWriter(self, incoming, final, 200, self.make_lease(), + FakeCanary()) + with self.assertRaises(DataTooLargeError): + bw.remote_write(offset, b"a" * length) + + @given( + maybe_overlapping_offset=strategies.integers(min_value=0, max_value=98), + maybe_overlapping_length=strategies.integers(min_value=1, max_value=100), + ) + def test_overlapping_writes_ok_if_matching( + self, maybe_overlapping_offset, maybe_overlapping_length + ): + """ + Writes that overlap with previous writes are OK when the content is the + same. + """ + length = 100 + expected_data = b"".join(bchr(i) for i in range(100)) + incoming, final = self.make_workdir("overlapping_writes_{}".format(uuid4())) + bw = BucketWriter( + self, incoming, final, length, self.make_lease(), + FakeCanary() + ) + # Three writes: 10-19, 30-39, 50-59. This allows for a bunch of holes. + bw.remote_write(10, expected_data[10:20]) + bw.remote_write(30, expected_data[30:40]) + bw.remote_write(50, expected_data[50:60]) + # Then, an overlapping write but with matching data: + bw.remote_write( + maybe_overlapping_offset, + expected_data[ + maybe_overlapping_offset:maybe_overlapping_offset + maybe_overlapping_length + ] + ) + # Now fill in the holes: + bw.remote_write(0, expected_data[0:10]) + bw.remote_write(20, expected_data[20:30]) + bw.remote_write(40, expected_data[40:50]) + bw.remote_write(60, expected_data[60:]) + bw.remote_close() + + br = BucketReader(self, bw.finalhome) + self.assertEqual(br.remote_read(0, length), expected_data) + + + @given( + maybe_overlapping_offset=strategies.integers(min_value=0, max_value=98), + maybe_overlapping_length=strategies.integers(min_value=1, max_value=100), + ) + def test_overlapping_writes_not_ok_if_different( + self, maybe_overlapping_offset, maybe_overlapping_length + ): + """ + Writes that overlap with previous writes fail with an exception if the + contents don't match. + """ + length = 100 + incoming, final = self.make_workdir("overlapping_writes_{}".format(uuid4())) + bw = BucketWriter( + self, incoming, final, length, self.make_lease(), + FakeCanary() + ) + # Three writes: 10-19, 30-39, 50-59. This allows for a bunch of holes. + bw.remote_write(10, b"1" * 10) + bw.remote_write(30, b"1" * 10) + bw.remote_write(50, b"1" * 10) + # Then, write something that might overlap with some of them, but + # conflicts. Then fill in holes left by first three writes. Conflict is + # inevitable. + with self.assertRaises(ConflictingWriteError): + bw.remote_write( + maybe_overlapping_offset, + b'X' * min(maybe_overlapping_length, length - maybe_overlapping_offset), + ) + bw.remote_write(0, b"1" * 10) + bw.remote_write(20, b"1" * 10) + bw.remote_write(40, b"1" * 10) + bw.remote_write(60, b"1" * 40) + def test_read_past_end_of_share_data(self): # test vector for immutable files (hard-coded contents of an immutable share # file): From cae989e8dec1ba76db46d2a4745aa3c3e65a7ee5 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 24 Sep 2021 11:54:03 -0400 Subject: [PATCH 219/269] News file. --- newsfragments/3801.bugfix | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/3801.bugfix diff --git a/newsfragments/3801.bugfix b/newsfragments/3801.bugfix new file mode 100644 index 000000000..504b3999d --- /dev/null +++ b/newsfragments/3801.bugfix @@ -0,0 +1 @@ +When uploading an immutable, overlapping writes that include conflicting data are rejected. In practice, this likely didn't happen in real-world usage. \ No newline at end of file From 6ef3811112ac5323af70aa8b3741c0075817f9fe Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 24 Sep 2021 11:54:08 -0400 Subject: [PATCH 220/269] Prevent conflicting overlapping writes. --- nix/tahoe-lafs.nix | 2 +- setup.py | 3 +++ src/allmydata/storage/immutable.py | 24 +++++++++++++++++++++--- 3 files changed, 25 insertions(+), 4 deletions(-) diff --git a/nix/tahoe-lafs.nix b/nix/tahoe-lafs.nix index 35b29f1cc..e08a2ab46 100644 --- a/nix/tahoe-lafs.nix +++ b/nix/tahoe-lafs.nix @@ -97,7 +97,7 @@ EOF setuptoolsTrial pyasn1 zope_interface service-identity pyyaml magic-wormhole treq eliot autobahn cryptography netifaces setuptools - future pyutil distro configparser + future pyutil distro configparser collections-extended ]; checkInputs = with python.pkgs; [ diff --git a/setup.py b/setup.py index e1d711ccf..8241898fe 100644 --- a/setup.py +++ b/setup.py @@ -137,6 +137,9 @@ install_requires = [ # Backported configparser for Python 2: "configparser ; python_version < '3.0'", + + # For the RangeMap datastructure. + "collections-extended", ] setup_requires = [ diff --git a/src/allmydata/storage/immutable.py b/src/allmydata/storage/immutable.py index 4b60d79f1..9e3a9622a 100644 --- a/src/allmydata/storage/immutable.py +++ b/src/allmydata/storage/immutable.py @@ -13,16 +13,20 @@ if PY2: import os, stat, struct, time +from collections_extended import RangeMap + from foolscap.api import Referenceable from zope.interface import implementer -from allmydata.interfaces import RIBucketWriter, RIBucketReader +from allmydata.interfaces import ( + RIBucketWriter, RIBucketReader, ConflictingWriteError, + DataTooLargeError, +) from allmydata.util import base32, fileutil, log from allmydata.util.assertutil import precondition from allmydata.util.hashutil import timing_safe_compare from allmydata.storage.lease import LeaseInfo -from allmydata.storage.common import UnknownImmutableContainerVersionError, \ - DataTooLargeError +from allmydata.storage.common import UnknownImmutableContainerVersionError # each share file (in storage/shares/$SI/$SHNUM) contains lease information # and share data. The share data is accessed by RIBucketWriter.write and @@ -217,6 +221,7 @@ class BucketWriter(Referenceable): # type: ignore # warner/foolscap#78 # also, add our lease to the file now, so that other ones can be # added by simultaneous uploaders self._sharefile.add_lease(lease_info) + self._already_written = RangeMap() def allocated_size(self): return self._max_size @@ -226,7 +231,20 @@ class BucketWriter(Referenceable): # type: ignore # warner/foolscap#78 precondition(not self.closed) if self.throw_out_all_data: return + + # Make sure we're not conflicting with existing data: + end = offset + len(data) + for (chunk_start, chunk_stop, _) in self._already_written.ranges(offset, end): + chunk_len = chunk_stop - chunk_start + actual_chunk = self._sharefile.read_share_data(chunk_start, chunk_len) + writing_chunk = data[chunk_start - offset:chunk_stop - offset] + if actual_chunk != writing_chunk: + raise ConflictingWriteError( + "Chunk {}-{} doesn't match already written data.".format(chunk_start, chunk_stop) + ) self._sharefile.write_share_data(offset, data) + + self._already_written.set(True, offset, end) self.ss.add_latency("write", time.time() - start) self.ss.count("write") From c1f8e9f8c763011cd73f1bb7c499fffe1ccf7171 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 24 Sep 2021 12:02:30 -0400 Subject: [PATCH 221/269] IStorageServer test for overlapping writes. --- src/allmydata/test/test_istorageserver.py | 26 ++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index a63189204..ef93dadf5 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -24,7 +24,7 @@ from testtools import skipIf from twisted.internet.defer import inlineCallbacks -from foolscap.api import Referenceable +from foolscap.api import Referenceable, RemoteException from allmydata.interfaces import IStorageServer from .common_system import SystemTestMixin @@ -248,11 +248,31 @@ class IStorageServerImmutableAPIsTestsMixin(object): (yield buckets[2].callRemote("read", 0, 1024)), b"3" * 512 + b"4" * 512 ) + @inlineCallbacks def test_overlapping_writes(self): """ - Overlapping writes in immutable uploads fail with ``OverlappingWriteError``. + Overlapping, non-identical writes in immutable uploads fail. """ - 1/0 + storage_index, renew_secret, cancel_secret = ( + new_storage_index(), + new_secret(), + new_secret(), + ) + (_, allocated) = yield self.storage_server.allocate_buckets( + storage_index, + renew_secret, + cancel_secret, + sharenums={0}, + allocated_size=30, + canary=Referenceable(), + ) + + yield allocated[0].callRemote("write", 0, b"1" * 10) + # Overlapping write that matches: + yield allocated[0].callRemote("write", 5, b"1" * 20) + # Overlapping write that doesn't match: + with self.assertRaises(RemoteException): + yield allocated[0].callRemote("write", 20, b"2" * 10) class _FoolscapMixin(SystemTestMixin): From 0b1082fc04ffa56a808bddb7fa519665a6135e66 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 24 Sep 2021 12:04:12 -0400 Subject: [PATCH 222/269] Fix lint. --- src/allmydata/storage/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/storage/common.py b/src/allmydata/storage/common.py index d72bb3fbc..e5563647f 100644 --- a/src/allmydata/storage/common.py +++ b/src/allmydata/storage/common.py @@ -14,7 +14,7 @@ import os.path from allmydata.util import base32 # Backwards compatibility. -from allmydata.interfaces import DataTooLargeError +from allmydata.interfaces import DataTooLargeError # noqa: F401 class UnknownMutableContainerVersionError(Exception): pass From 96acb1419977bd64898beb08e93a3c6e58505e8f Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Fri, 24 Sep 2021 13:48:07 -0400 Subject: [PATCH 223/269] Document impact of semantic changes on HTTP protocol. --- docs/proposed/http-storage-node-protocol.rst | 28 +++++++++++--------- 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index a84d62176..bf01bcf12 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -497,22 +497,26 @@ If any one of these requests fails then at most 128KiB of upload work needs to b The server must recognize when all of the data has been received and mark the share as complete (which it can do because it was informed of the size when the storage index was initialized). -Clients should upload chunks in re-assembly order. * When a chunk that does not complete the share is successfully uploaded the response is ``OK``. * When the chunk that completes the share is successfully uploaded the response is ``CREATED``. -* If the *Content-Range* for a request covers part of the share that has already been uploaded the response is ``CONFLICT``. - The response body indicates the range of share data that has yet to be uploaded. - That is:: +* If the *Content-Range* for a request covers part of the share that has already, + and the data does not match already written data, + the response is ``CONFLICT``. + At this point the only thing to do is abort the upload and start from scratch (see below). - { "required": - [ { "begin": - , "end": - } - , - ... - ] - } +``PUT /v1/immutable/:storage_index/:share_number/abort`` +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +This cancels an *in-progress* upload. + +The response code: + +* When the upload is still in progress and therefore the abort has succeeded, + the response is ``OK``. + Future uploads can start from scratch with no pre-existing upload state stored on the server. +* If the uploaded has already finished, the response is 405 (Method Not Allowed) + and no change is made. ``POST /v1/immutable/:storage_index/:share_number/corrupt`` From b1b64c787ebd84d12df754d7455c46a0062adc04 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 27 Sep 2021 15:00:52 -0400 Subject: [PATCH 224/269] Add more randomness. --- src/allmydata/test/test_istorageserver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index e35fbf29d..355a07841 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -300,7 +300,7 @@ class IStorageServerImmutableAPIsTestsMixin(object): canary=Referenceable(), ) - total_data = _randbytes(256) * 17 + total_data = _randbytes(256 * 17) yield allocated[0].callRemote("write", 0, total_data) yield allocated[0].callRemote("close") From da937cef9e74f67c0c6abe388ae8f14075276059 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 27 Sep 2021 15:04:04 -0400 Subject: [PATCH 225/269] Correct docstring. --- src/allmydata/interfaces.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/allmydata/interfaces.py b/src/allmydata/interfaces.py index 0283b443a..7f7564550 100644 --- a/src/allmydata/interfaces.py +++ b/src/allmydata/interfaces.py @@ -353,8 +353,10 @@ class IStorageServer(Interface): :see: ``RIStorageServer.slot_testv_readv_and_writev`` While the interface mostly matches, test vectors are simplified. - Instead of a tuple ``(start, offset, operator, data)`` they are just - ``(start, data_length, data)`` with operator implicitly being ``b"eq"``. + Instead of a tuple ``(offset, read_size, operator, expected_data)`` in + the original, for this method you need only pass in + ``(offset, read_size, expected_data)``, with the operator implicitly + being ``b"eq"``. """ def advise_corrupt_share( From c740da4acde9f70a6a765235eb1dd061d730dcc2 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 27 Sep 2021 15:09:24 -0400 Subject: [PATCH 226/269] Explain what happens in reads past the end. --- docs/proposed/http-storage-node-protocol.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index 35811cad0..9d419657b 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -625,6 +625,9 @@ For example:: } } +A test vector or read vector that read beyond the boundaries of existing data will return nothing for any bytes past the end. +As a result, if there is no data at all, an empty bytestring is returned no matter what the offset or length. + Reading ~~~~~~~ From 38e449aceb611661e899b4dec0babf166ddd5f09 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 27 Sep 2021 16:44:43 -0400 Subject: [PATCH 227/269] Add collections-extended. --- nix/collections-extended.nix | 19 +++++++++++++++++++ nix/overlays.nix | 3 +++ 2 files changed, 22 insertions(+) create mode 100644 nix/collections-extended.nix diff --git a/nix/collections-extended.nix b/nix/collections-extended.nix new file mode 100644 index 000000000..3f1ad165a --- /dev/null +++ b/nix/collections-extended.nix @@ -0,0 +1,19 @@ +{ lib, buildPythonPackage, fetchPypi }: +buildPythonPackage rec { + pname = "collections-extended"; + version = "1.0.3"; + + src = fetchPypi { + inherit pname version; + sha256 = "0lb69x23asd68n0dgw6lzxfclavrp2764xsnh45jm97njdplznkw"; + }; + + # Tests aren't in tarball, for 1.0.3 at least. + doCheck = false; + + meta = with lib; { + homepage = https://github.com/mlenzen/collections-extended; + description = "Extra Python Collections - bags (multisets), setlists (unique list / indexed set), RangeMap and IndexedDict"; + license = licenses.asl20; + }; +} diff --git a/nix/overlays.nix b/nix/overlays.nix index 2bf58575e..aae808131 100644 --- a/nix/overlays.nix +++ b/nix/overlays.nix @@ -18,6 +18,9 @@ self: super: { # Need a newer version of Twisted, too. twisted = python-super.callPackage ./twisted.nix { }; + + # collections-extended is not part of nixpkgs at this time. + collections-extended = python-super.callPackage ./collections-extended.nix }; }; } From 4e6438352ad95fe332bce67f817aa49d91a20001 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 27 Sep 2021 16:45:01 -0400 Subject: [PATCH 228/269] Extend to end. --- docs/proposed/http-storage-node-protocol.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index bf01bcf12..d91687960 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -506,7 +506,7 @@ The server must recognize when all of the data has been received and mark the sh At this point the only thing to do is abort the upload and start from scratch (see below). ``PUT /v1/immutable/:storage_index/:share_number/abort`` -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! This cancels an *in-progress* upload. From 60cb3c08836603016f4fb9e59e075cb34a92f1cb Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 27 Sep 2021 16:52:25 -0400 Subject: [PATCH 229/269] Restore range result. --- docs/proposed/http-storage-node-protocol.rst | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index d91687960..2984b5c6d 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -499,6 +499,18 @@ The server must recognize when all of the data has been received and mark the sh (which it can do because it was informed of the size when the storage index was initialized). * When a chunk that does not complete the share is successfully uploaded the response is ``OK``. + The response body indicates the range of share data that has yet to be uploaded. + That is:: + + { "required": + [ { "begin": + , "end": + } + , + ... + ] + } + * When the chunk that completes the share is successfully uploaded the response is ``CREATED``. * If the *Content-Range* for a request covers part of the share that has already, and the data does not match already written data, From de1a7d7fcee4a400c791f28f1afb32493bb3e43e Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 27 Sep 2021 16:56:24 -0400 Subject: [PATCH 230/269] A more explicit test for successful overlapping. --- src/allmydata/test/test_istorageserver.py | 39 ++++++++++++++++++++--- 1 file changed, 34 insertions(+), 5 deletions(-) diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index ef93dadf5..9e9a41d35 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -249,9 +249,10 @@ class IStorageServerImmutableAPIsTestsMixin(object): ) @inlineCallbacks - def test_overlapping_writes(self): + def test_non_matching_overlapping_writes(self): """ - Overlapping, non-identical writes in immutable uploads fail. + When doing overlapping writes in immutable uploads, non-matching writes + fail. """ storage_index, renew_secret, cancel_secret = ( new_storage_index(), @@ -267,13 +268,41 @@ class IStorageServerImmutableAPIsTestsMixin(object): canary=Referenceable(), ) - yield allocated[0].callRemote("write", 0, b"1" * 10) - # Overlapping write that matches: - yield allocated[0].callRemote("write", 5, b"1" * 20) + yield allocated[0].callRemote("write", 0, b"1" * 25) # Overlapping write that doesn't match: with self.assertRaises(RemoteException): yield allocated[0].callRemote("write", 20, b"2" * 10) + @inlineCallbacks + def test_matching_overlapping_writes(self): + """ + When doing overlapping writes in immutable uploads, matching writes + succeed. + """ + storage_index, renew_secret, cancel_secret = ( + new_storage_index(), + new_secret(), + new_secret(), + ) + (_, allocated) = yield self.storage_server.allocate_buckets( + storage_index, + renew_secret, + cancel_secret, + sharenums={0}, + allocated_size=25, + canary=Referenceable(), + ) + + yield allocated[0].callRemote("write", 0, b"1" * 10) + # Overlapping write that matches: + yield allocated[0].callRemote("write", 5, b"1" * 20) + yield allocated[0].callRemote("close") + + buckets = yield self.storage_server.get_buckets(storage_index) + self.assertEqual(set(buckets.keys()), {0}) + + self.assertEqual((yield buckets[0].callRemote("read", 0, 25)), b"1" * 25) + class _FoolscapMixin(SystemTestMixin): """Run tests on Foolscap version of ``IStorageServer.""" From f66f3e64ada92a37c15d7d2e41c7a37cc78c2c41 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 27 Sep 2021 16:58:18 -0400 Subject: [PATCH 231/269] Fix syntax. --- nix/overlays.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nix/overlays.nix b/nix/overlays.nix index aae808131..14a47ca5a 100644 --- a/nix/overlays.nix +++ b/nix/overlays.nix @@ -20,7 +20,7 @@ self: super: { twisted = python-super.callPackage ./twisted.nix { }; # collections-extended is not part of nixpkgs at this time. - collections-extended = python-super.callPackage ./collections-extended.nix + collections-extended = python-super.callPackage ./collections-extended.nix { }; }; }; } From 9f80435b41b246c7675a5586292d17931e392791 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 27 Sep 2021 17:04:22 -0400 Subject: [PATCH 232/269] Update to new interface. --- src/allmydata/test/test_istorageserver.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index 98c096b3a..2aad5786c 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -458,7 +458,7 @@ class IStorageServerMutableAPIsTestsMixin(object): secrets, tw_vectors={ 0: ( - [(0, 3, b"eq", b"1" * 3), (3, 4, b"eq", b"1" * 4)], + [(0, 3, b"1" * 3), (3, 4, b"1" * 4)], [(0, b"2" * 7)], 7, ), @@ -479,7 +479,7 @@ class IStorageServerMutableAPIsTestsMixin(object): storage_index, secrets, tw_vectors={ - 0: ([(0, 7, b"eq", b"1" * 7)], [(0, b"3" * 7)], 7), + 0: ([(0, 7, b"1" * 7)], [(0, b"3" * 7)], 7), }, r_vector=[], ) From 914ca567758bad81ae91844a525a143d975fe638 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 27 Sep 2021 17:05:03 -0400 Subject: [PATCH 233/269] TODOs. --- src/allmydata/test/test_istorageserver.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index 2aad5786c..615a1c24f 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -568,7 +568,12 @@ class IStorageServerMutableAPIsTestsMixin(object): ) self.assertEqual(reads, {}) + # TODO detection of empty/missing shares with length 1 reads that expect + # empty bytestring. + # TODO what else? + + class _FoolscapMixin(SystemTestMixin): """Run tests on Foolscap version of ``IStorageServer.""" From c668e342c294a1c479f2f95ad3b020861f7c94a3 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 28 Sep 2021 09:51:40 -0400 Subject: [PATCH 234/269] More STARAW tests. --- src/allmydata/test/test_istorageserver.py | 71 +++++++++++++++++++++-- 1 file changed, 65 insertions(+), 6 deletions(-) diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index 615a1c24f..c606afcc1 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -352,6 +352,7 @@ class IStorageServerMutableAPIsTestsMixin(object): ``STARAW`` is short for ``slot_testv_and_readv_and_writev``. """ + def new_secrets(self): """Return a 3-tuple of secrets for STARAW calls.""" return (new_secret(), new_secret(), new_secret()) @@ -434,7 +435,7 @@ class IStorageServerMutableAPIsTestsMixin(object): self.assertEqual(reads, {0: [b"X" * 7]}) @inlineCallbacks - def test_SATRAW_writen_happens_only_if_test_matches(self): + def test_SATRAW_writes_happens_only_if_test_matches(self): """ If a ``IStorageServer.slot_testv_and_readv_and_writev`` includes both a test and a write, the write succeeds if the test matches, and fails if @@ -492,6 +493,69 @@ class IStorageServerMutableAPIsTestsMixin(object): ) self.assertEqual(reads, {0: [b"2" * 7]}) + @inlineCallbacks + def test_SATRAW_tests_past_end_of_data(self): + """ + If a ``IStorageServer.slot_testv_and_readv_and_writev`` includes a test + vector that reads past the end of the data, the result is limited to + actual available data. + """ + secrets = self.new_secrets() + storage_index = new_storage_index() + + # Since there is no data on server, the test vector will return empty + # string, which matches expected result, so write will succeed. + (written, _) = yield self.staraw( + storage_index, + secrets, + tw_vectors={ + 0: ([(0, 10, b"")], [(0, b"1" * 7)], 7), + }, + r_vector=[], + ) + self.assertEqual(written, True) + + # Now the test vector is a 10-read off of a 7-byte value, but expected + # value is still 7 bytes, so the write will again succeed. + (written, _) = yield self.staraw( + storage_index, + secrets, + tw_vectors={ + 0: ([(0, 10, b"1" * 7)], [(0, b"2" * 7)], 7), + }, + r_vector=[], + ) + self.assertEqual(written, True) + + @inlineCallbacks + def test_SATRAW_reads_past_end_of_data(self): + """ + If a ``IStorageServer.slot_testv_and_readv_and_writev`` reads past the + end of the data, the result is limited to actual available data. + """ + secrets = self.new_secrets() + storage_index = new_storage_index() + + # Write some data + (written, _) = yield self.staraw( + storage_index, + secrets, + tw_vectors={ + 0: ([], [(0, b"12345")], 5), + }, + r_vector=[], + ) + self.assertEqual(written, True) + + # Reads past end. + (_, reads) = yield self.staraw( + storage_index, + secrets, + tw_vectors={}, + r_vector=[(0, 100), (2, 50)], + ) + self.assertEqual(reads, {0: [b"12345", b"345"]}) + @inlineCallbacks def test_STARAW_write_enabler_must_match(self): """ @@ -568,12 +632,7 @@ class IStorageServerMutableAPIsTestsMixin(object): ) self.assertEqual(reads, {}) - # TODO detection of empty/missing shares with length 1 reads that expect - # empty bytestring. - # TODO what else? - - class _FoolscapMixin(SystemTestMixin): """Run tests on Foolscap version of ``IStorageServer.""" From 8e5928a39e6f0e982871a27d30a34a614fff9e38 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 28 Sep 2021 10:03:49 -0400 Subject: [PATCH 235/269] News file. --- newsfragments/3805.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3805.minor diff --git a/newsfragments/3805.minor b/newsfragments/3805.minor new file mode 100644 index 000000000..e69de29bb From 7e232f602a97592dc62ffa1033a84fcf6090cc75 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 28 Sep 2021 10:34:57 -0400 Subject: [PATCH 236/269] Switch to PATCH. --- docs/proposed/http-storage-node-protocol.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index 9d419657b..56f383e2c 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -482,7 +482,7 @@ The response includes ``already-have`` and ``allocated`` for two reasons: This might be because a server has become unavailable and a remaining server needs to store more shares for the upload. It could also just be that the client's preferred servers have changed. -``PUT /v1/immutable/:storage_index/:share_number`` +``PATCH /v1/immutable/:storage_index/:share_number`` !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! Write data for the indicated share. @@ -668,19 +668,19 @@ Immutable Data #. Upload the content for immutable share ``7``:: - PUT /v1/immutable/AAAAAAAAAAAAAAAA/7 + PATCH /v1/immutable/AAAAAAAAAAAAAAAA/7 Content-Range: bytes 0-15/48 200 OK - PUT /v1/immutable/AAAAAAAAAAAAAAAA/7 + PATCH /v1/immutable/AAAAAAAAAAAAAAAA/7 Content-Range: bytes 16-31/48 200 OK - PUT /v1/immutable/AAAAAAAAAAAAAAAA/7 + PATCH /v1/immutable/AAAAAAAAAAAAAAAA/7 Content-Range: bytes 32-47/48 From ed290caeeca024c73bb15982709e4caf28a10014 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 28 Sep 2021 10:35:10 -0400 Subject: [PATCH 237/269] News file. --- newsfragments/3806.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3806.minor diff --git a/newsfragments/3806.minor b/newsfragments/3806.minor new file mode 100644 index 000000000..e69de29bb From bb85a9d2cf046dcd3d49312e29ffe4cd5ae6a672 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 28 Sep 2021 10:37:16 -0400 Subject: [PATCH 238/269] !! --- docs/proposed/http-storage-node-protocol.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index 56f383e2c..e3bd48128 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -483,7 +483,7 @@ The response includes ``already-have`` and ``allocated`` for two reasons: It could also just be that the client's preferred servers have changed. ``PATCH /v1/immutable/:storage_index/:share_number`` -!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! Write data for the indicated share. The share number must belong to the storage index. From 9970559019451c6a5bf68d1f4d06baeac430092e Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 28 Sep 2021 10:46:21 -0400 Subject: [PATCH 239/269] Tests for slot_readv. --- src/allmydata/test/test_istorageserver.py | 62 +++++++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index c606afcc1..931cf6ed6 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -632,6 +632,68 @@ class IStorageServerMutableAPIsTestsMixin(object): ) self.assertEqual(reads, {}) + @inlineCallbacks + def test_slot_readv(self): + """ + Data written with ``IStorageServer.slot_testv_and_readv_and_writev()`` + can be read using ``IStorageServer.slot_readv()``. Reads can't go past + the end of the data. + """ + secrets = self.new_secrets() + storage_index = new_storage_index() + (written, _) = yield self.staraw( + storage_index, + secrets, + tw_vectors={ + 0: ([], [(0, b"abcdefg")], 7), + 1: ([], [(0, b"0123"), (4, b"456")], 7), + 2: ([], [(0, b"0123")], 4), + }, + r_vector=[], + ) + self.assertEqual(written, True) + + reads = yield self.storage_server.slot_readv( + storage_index, + shares=[0, 1], + # Whole thing, partial, going beyond the edge, completely outside + # range: + readv=[(0, 7), (2, 3), (6, 8), (100, 10)], + ) + self.assertEqual( + reads, + {0: [b"abcdefg", b"cde", b"g", b""], 1: [b"0123456", b"234", b"6", b""]}, + ) + + @inlineCallbacks + def test_slot_readv_no_shares(self): + """ + With no shares given, ``IStorageServer.slot_readv()`` reads from all shares. + """ + secrets = self.new_secrets() + storage_index = new_storage_index() + (written, _) = yield self.staraw( + storage_index, + secrets, + tw_vectors={ + 0: ([], [(0, b"abcdefg")], 7), + 1: ([], [(0, b"0123456")], 7), + 2: ([], [(0, b"9876543")], 7), + }, + r_vector=[], + ) + self.assertEqual(written, True) + + reads = yield self.storage_server.slot_readv( + storage_index, + shares=[], + readv=[(0, 7)], + ) + self.assertEqual( + reads, + {0: [b"abcdefg"], 1: [b"0123456"], 2: [b"9876543"]}, + ) + class _FoolscapMixin(SystemTestMixin): """Run tests on Foolscap version of ``IStorageServer.""" From a02d5f4c9cb81e46c461dda417be327b2de604a9 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 28 Sep 2021 13:02:01 -0400 Subject: [PATCH 240/269] Just stick to current behavior. --- src/allmydata/test/test_istorageserver.py | 44 +++-------------------- 1 file changed, 4 insertions(+), 40 deletions(-) diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index 7c6090980..be327770e 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -98,13 +98,10 @@ class IStorageServerImmutableAPIsTestsMixin(object): # We validate the bucket objects' interface in a later test. @inlineCallbacks - @skipIf(True, "https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3793") def test_allocate_buckets_repeat(self): """ - allocate_buckets() with the same storage index returns the same result, - because the shares have not been written to. - - This fails due to https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3793 + allocate_buckets() with the same storage index does not return + work-in-progress buckets, but will add any newly added buckets. """ storage_index, renew_secret, cancel_secret = ( new_storage_index(), @@ -115,7 +112,7 @@ class IStorageServerImmutableAPIsTestsMixin(object): storage_index, renew_secret, cancel_secret, - sharenums=set(range(5)), + sharenums=set(range(4)), allocated_size=1024, canary=Referenceable(), ) @@ -128,40 +125,7 @@ class IStorageServerImmutableAPIsTestsMixin(object): Referenceable(), ) self.assertEqual(already_got, already_got2) - self.assertEqual(set(allocated.keys()), set(allocated2.keys())) - - @skipIf(True, "https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3793") - @inlineCallbacks - def test_allocate_buckets_more_sharenums(self): - """ - allocate_buckets() with the same storage index but more sharenums - acknowledges the extra shares don't exist. - - Fails due to https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3793 - """ - storage_index, renew_secret, cancel_secret = ( - new_storage_index(), - new_secret(), - new_secret(), - ) - yield self.storage_server.allocate_buckets( - storage_index, - renew_secret, - cancel_secret, - sharenums=set(range(5)), - allocated_size=1024, - canary=Referenceable(), - ) - (already_got2, allocated2) = yield self.storage_server.allocate_buckets( - storage_index, - renew_secret, - cancel_secret, - sharenums=set(range(7)), - allocated_size=1024, - canary=Referenceable(), - ) - self.assertEqual(already_got2, set()) # none were fully written - self.assertEqual(set(allocated2.keys()), set(range(7))) + self.assertEqual(set(allocated2.keys()), {4}) @inlineCallbacks def test_written_shares_are_allocated(self): From 2b1502eff601a5c43b2d7f2fb9ad6de4959a4fc5 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 28 Sep 2021 13:02:24 -0400 Subject: [PATCH 241/269] News file. --- newsfragments/3793.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3793.minor diff --git a/newsfragments/3793.minor b/newsfragments/3793.minor new file mode 100644 index 000000000..e69de29bb From e64c397fc5d22052c6268e14704041ed0c85f7f6 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Tue, 28 Sep 2021 13:51:31 -0400 Subject: [PATCH 242/269] WIP disconnection test. --- src/allmydata/test/test_istorageserver.py | 75 +++++++++++++++++++++-- 1 file changed, 69 insertions(+), 6 deletions(-) diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index be327770e..daf264286 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -20,9 +20,9 @@ if PY2: from random import Random -from testtools import skipIf - from twisted.internet.defer import inlineCallbacks +from twisted.internet import reactor +from twisted.internet.task import deferLater from foolscap.api import Referenceable, RemoteException @@ -77,6 +77,10 @@ class IStorageServerImmutableAPIsTestsMixin(object): Tests for ``IStorageServer``'s immutable APIs. ``self.storage_server`` is expected to provide ``IStorageServer``. + + ``self.disconnect()`` should disconnect and then reconnect, creating a new + ``self.storage_server``. Some implementations may wish to skip tests using + this; HTTP has no notion of disconnection. """ @inlineCallbacks @@ -100,7 +104,7 @@ class IStorageServerImmutableAPIsTestsMixin(object): @inlineCallbacks def test_allocate_buckets_repeat(self): """ - allocate_buckets() with the same storage index does not return + ``IStorageServer.allocate_buckets()`` with the same storage index does not return work-in-progress buckets, but will add any newly added buckets. """ storage_index, renew_secret, cancel_secret = ( @@ -127,6 +131,45 @@ class IStorageServerImmutableAPIsTestsMixin(object): self.assertEqual(already_got, already_got2) self.assertEqual(set(allocated2.keys()), {4}) + @inlineCallbacks + def test_disconnection(self): + """ + If we disconnect in the middle of writing to a bucket, all data is + wiped, and it's even possible to write different data to the bucket + (don't do that though, mostly it's just a good way to test that the + data really was wiped). + """ + storage_index, renew_secret, cancel_secret = ( + new_storage_index(), + new_secret(), + new_secret(), + ) + (_, allocated) = yield self.storage_server.allocate_buckets( + storage_index, + renew_secret, + cancel_secret, + sharenums={0}, + allocated_size=1024, + canary=Referenceable(), + ) + + # Bucket 1 is fully written in one go. + yield allocated[0].callRemote("write", 0, b"1" * 1024) + + # Disconnect: + yield self.disconnect() + + # Write different data with no complaint: + (_, allocated) = yield self.storage_server.allocate_buckets( + storage_index, + renew_secret, + cancel_secret, + sharenums={0}, + allocated_size=1024, + canary=Referenceable(), + ) + yield allocated[0].callRemote("write", 0, b"2" * 1024) + @inlineCallbacks def test_written_shares_are_allocated(self): """ @@ -359,15 +402,16 @@ class IStorageServerImmutableAPIsTestsMixin(object): class _FoolscapMixin(SystemTestMixin): """Run tests on Foolscap version of ``IStorageServer.""" + def _get_native_server(self): + return next(iter(self.clients[0].storage_broker.get_known_servers())) + @inlineCallbacks def setUp(self): AsyncTestCase.setUp(self) self.basedir = "test_istorageserver/" + self.id() yield SystemTestMixin.setUp(self) yield self.set_up_nodes(1) - self.storage_server = next( - iter(self.clients[0].storage_broker.get_known_servers()) - ).get_storage_server() + self.storage_server = self._get_native_server().get_storage_server() self.assertTrue(IStorageServer.providedBy(self.storage_server)) @inlineCallbacks @@ -375,6 +419,25 @@ class _FoolscapMixin(SystemTestMixin): AsyncTestCase.tearDown(self) yield SystemTestMixin.tearDown(self) + @inlineCallbacks + def disconnect(self): + """ + Disconnect and then reconnect with a new ``IStorageServer``. + """ + current = self.storage_server + self._get_native_server()._rref.tracker.broker.transport.loseConnection() + for i in range(100000): + yield deferLater(reactor, 0.001) + import pdb + + pdb.set_trace() + new = self._get_native_server().get_storage_server() + if new is not None and new is not current: + self.storage_server = new + return + + raise RuntimeError("Failed to reconnect") + class FoolscapSharedAPIsTests( _FoolscapMixin, IStorageServerSharedAPIsTestsMixin, AsyncTestCase From 51e8b5e197e07ec2247c75212e0df342bc3a091d Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 29 Sep 2021 11:17:33 -0400 Subject: [PATCH 243/269] Disconnection test works now. --- src/allmydata/test/test_istorageserver.py | 22 +++++++--------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index daf264286..25b814237 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -135,9 +135,10 @@ class IStorageServerImmutableAPIsTestsMixin(object): def test_disconnection(self): """ If we disconnect in the middle of writing to a bucket, all data is - wiped, and it's even possible to write different data to the bucket - (don't do that though, mostly it's just a good way to test that the - data really was wiped). + wiped, and it's even possible to write different data to the bucket. + + (In the real world one shouldn't do that, but writing different data is + a good way to test that the original data really was wiped.) """ storage_index, renew_secret, cancel_secret = ( new_storage_index(), @@ -425,18 +426,9 @@ class _FoolscapMixin(SystemTestMixin): Disconnect and then reconnect with a new ``IStorageServer``. """ current = self.storage_server - self._get_native_server()._rref.tracker.broker.transport.loseConnection() - for i in range(100000): - yield deferLater(reactor, 0.001) - import pdb - - pdb.set_trace() - new = self._get_native_server().get_storage_server() - if new is not None and new is not current: - self.storage_server = new - return - - raise RuntimeError("Failed to reconnect") + yield self.bounce_client(0) + self.storage_server = self._get_native_server().get_storage_server() + assert self.storage_server is not current class FoolscapSharedAPIsTests( From a4153b71256db2dd4094e2e21324dd2b8c9cfcab Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 29 Sep 2021 11:56:04 -0400 Subject: [PATCH 244/269] Implementation plan. --- src/allmydata/storage/immutable.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/allmydata/storage/immutable.py b/src/allmydata/storage/immutable.py index 9e3a9622a..c2b190e01 100644 --- a/src/allmydata/storage/immutable.py +++ b/src/allmydata/storage/immutable.py @@ -204,6 +204,16 @@ class ShareFile(object): self.unlink() return space_freed +# TODOs +# Batch 1: +# - bucketwriter dict in the server, to persist them + TEST of persistence +# - aborting bucketwriter removes it from server persistent + TEST +# - get rid of disconnect notification (probably no test, rely on existing?) +# - add bucketwriter cancellation to remote_allocate_buckets() (probably rely on existing tests) +# Batch 2: +# - scheduled events for aborting bucketwriter + TEST +# - bucketwriter writes delay cancellation + TEST + @implementer(RIBucketWriter) class BucketWriter(Referenceable): # type: ignore # warner/foolscap#78 From 8fb6afee1bc2ff5509695c5d53b2d5dd3d72ed08 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 29 Sep 2021 13:42:17 -0400 Subject: [PATCH 245/269] Refactor BucketWriters such that disconnection can be limited Foolscap. --- src/allmydata/storage/immutable.py | 14 +++----- src/allmydata/storage/server.py | 54 ++++++++++++++++++++++++------ src/allmydata/test/common_util.py | 5 +++ src/allmydata/test/test_storage.py | 34 ++++++------------- 4 files changed, 65 insertions(+), 42 deletions(-) diff --git a/src/allmydata/storage/immutable.py b/src/allmydata/storage/immutable.py index c2b190e01..d3b6ce875 100644 --- a/src/allmydata/storage/immutable.py +++ b/src/allmydata/storage/immutable.py @@ -208,8 +208,9 @@ class ShareFile(object): # Batch 1: # - bucketwriter dict in the server, to persist them + TEST of persistence # - aborting bucketwriter removes it from server persistent + TEST -# - get rid of disconnect notification (probably no test, rely on existing?) -# - add bucketwriter cancellation to remote_allocate_buckets() (probably rely on existing tests) +# - disconnect still aborts _for Foolscap only_ +# - existing in-use buckets are not returned _for Foolscap only_ +# - this implies splitting remote_allocate_buckets into generic and Foolscap-y parts # Batch 2: # - scheduled events for aborting bucketwriter + TEST # - bucketwriter writes delay cancellation + TEST @@ -218,13 +219,11 @@ class ShareFile(object): @implementer(RIBucketWriter) class BucketWriter(Referenceable): # type: ignore # warner/foolscap#78 - def __init__(self, ss, incominghome, finalhome, max_size, lease_info, canary): + def __init__(self, ss, incominghome, finalhome, max_size, lease_info): self.ss = ss self.incominghome = incominghome self.finalhome = finalhome self._max_size = max_size # don't allow the client to write more than this - self._canary = canary - self._disconnect_marker = canary.notifyOnDisconnect(self._disconnected) self.closed = False self.throw_out_all_data = False self._sharefile = ShareFile(incominghome, create=True, max_size=max_size) @@ -290,22 +289,19 @@ class BucketWriter(Referenceable): # type: ignore # warner/foolscap#78 pass self._sharefile = None self.closed = True - self._canary.dontNotifyOnDisconnect(self._disconnect_marker) filelen = os.stat(self.finalhome)[stat.ST_SIZE] self.ss.bucket_writer_closed(self, filelen) self.ss.add_latency("close", time.time() - start) self.ss.count("close") - def _disconnected(self): + def disconnected(self): if not self.closed: self._abort() def remote_abort(self): log.msg("storage: aborting sharefile %s" % self.incominghome, facility="tahoe.storage", level=log.UNUSUAL) - if not self.closed: - self._canary.dontNotifyOnDisconnect(self._disconnect_marker) self._abort() self.ss.count("abort") diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py index f4996756e..92b3d2a1b 100644 --- a/src/allmydata/storage/server.py +++ b/src/allmydata/storage/server.py @@ -11,13 +11,14 @@ if PY2: # Omit open() to get native behavior where open("w") always accepts native # strings. Omit bytes so we don't leak future's custom bytes. from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, pow, round, super, dict, list, object, range, str, max, min # noqa: F401 - +else: + from typing import Dict import os, re, struct, time -import weakref import six from foolscap.api import Referenceable +from foolscap.ipb import IRemoteReference from twisted.application import service from zope.interface import implementer @@ -89,7 +90,6 @@ class StorageServer(service.MultiService, Referenceable): self.incomingdir = os.path.join(sharedir, 'incoming') self._clean_incomplete() fileutil.make_dirs(self.incomingdir) - self._active_writers = weakref.WeakKeyDictionary() log.msg("StorageServer created", facility="tahoe.storage") if reserved_space: @@ -121,6 +121,15 @@ class StorageServer(service.MultiService, Referenceable): self.lease_checker.setServiceParent(self) self._get_current_time = get_current_time + # Currently being-written Bucketwriters. TODO Can probably refactor so + # active_writers is unnecessary, do as second pass. + # Map BucketWriter -> (storage_index, share_num) + self._active_writers = {} # type: Dict[BucketWriter, (bytes,int)] + # Map (storage_index, share_num) -> BucketWriter: + self._bucket_writers = {} # type: Dict[(bytes, int),BucketWriter] + # Canaries and disconnect markers for BucketWriters created via Foolscap: + self._bucket_writer_disconnect_markers = {} # type: Dict[BucketWriter,(IRemoteReference, object)] + def __repr__(self): return "" % (idlib.shortnodeid_b2a(self.my_nodeid),) @@ -263,10 +272,14 @@ class StorageServer(service.MultiService, Referenceable): } return version - def remote_allocate_buckets(self, storage_index, - renew_secret, cancel_secret, - sharenums, allocated_size, - canary, owner_num=0): + def allocate_buckets(self, storage_index, + renew_secret, cancel_secret, + sharenums, allocated_size, + include_in_progress, + owner_num=0): + """ + Generic bucket allocation API. + """ # owner_num is not for clients to set, but rather it should be # curried into the PersonalStorageServer instance that is dedicated # to a particular owner. @@ -315,6 +328,7 @@ class StorageServer(service.MultiService, Referenceable): # great! we already have it. easy. pass elif os.path.exists(incominghome): + # TODO use include_in_progress # Note that we don't create BucketWriters for shnums that # have a partial share (in incoming/), so if a second upload # occurs while the first is still in progress, the second @@ -323,11 +337,12 @@ class StorageServer(service.MultiService, Referenceable): elif (not limited) or (remaining_space >= max_space_per_bucket): # ok! we need to create the new share file. bw = BucketWriter(self, incominghome, finalhome, - max_space_per_bucket, lease_info, canary) + max_space_per_bucket, lease_info) if self.no_storage: bw.throw_out_all_data = True bucketwriters[shnum] = bw - self._active_writers[bw] = 1 + self._active_writers[bw] = (storage_index, shnum) + self._bucket_writers[(storage_index, shnum)] = bw if limited: remaining_space -= max_space_per_bucket else: @@ -340,6 +355,21 @@ class StorageServer(service.MultiService, Referenceable): self.add_latency("allocate", self._get_current_time() - start) return alreadygot, bucketwriters + def remote_allocate_buckets(self, storage_index, + renew_secret, cancel_secret, + sharenums, allocated_size, + canary, owner_num=0): + """Foolscap-specific ``allocate_buckets()`` API.""" + alreadygot, bucketwriters = self.allocate_buckets( + storage_index, renew_secret, cancel_secret, sharenums, allocated_size, + include_in_progress=False, owner_num=owner_num, + ) + # Abort BucketWriters if disconnection happens. + for bw in bucketwriters.values(): + disconnect_marker = canary.notifyOnDisconnect(bw.disconnected) + self._bucket_writer_disconnect_markers[bw] = (canary, disconnect_marker) + return alreadygot, bucketwriters + def _iter_share_files(self, storage_index): for shnum, filename in self._get_bucket_shares(storage_index): with open(filename, 'rb') as f: @@ -383,7 +413,11 @@ class StorageServer(service.MultiService, Referenceable): def bucket_writer_closed(self, bw, consumed_size): if self.stats_provider: self.stats_provider.count('storage_server.bytes_added', consumed_size) - del self._active_writers[bw] + storage_index, shnum = self._active_writers.pop(bw) + del self._bucket_writers[(storage_index, shnum)] + if bw in self._bucket_writer_disconnect_markers: + canary, disconnect_marker = self._bucket_writer_disconnect_markers.pop(bw) + canary.dontNotifyOnDisconnect(disconnect_marker) def _get_bucket_shares(self, storage_index): """Return a list of (shnum, pathname) tuples for files that hold diff --git a/src/allmydata/test/common_util.py b/src/allmydata/test/common_util.py index b5229ca11..de8d774b3 100644 --- a/src/allmydata/test/common_util.py +++ b/src/allmydata/test/common_util.py @@ -314,6 +314,11 @@ class FakeCanary(object): def getPeer(self): return "" + # For use by tests: + def disconnected(self): + for (f, args, kwargs) in list(self.disconnectors.values()): + f(*args, **kwargs) + class ShouldFailMixin(object): diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index 60173cf75..3b36f293f 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -129,8 +129,7 @@ class Bucket(unittest.TestCase): def test_create(self): incoming, final = self.make_workdir("test_create") - bw = BucketWriter(self, incoming, final, 200, self.make_lease(), - FakeCanary()) + bw = BucketWriter(self, incoming, final, 200, self.make_lease()) bw.remote_write(0, b"a"*25) bw.remote_write(25, b"b"*25) bw.remote_write(50, b"c"*25) @@ -139,8 +138,7 @@ class Bucket(unittest.TestCase): def test_readwrite(self): incoming, final = self.make_workdir("test_readwrite") - bw = BucketWriter(self, incoming, final, 200, self.make_lease(), - FakeCanary()) + bw = BucketWriter(self, incoming, final, 200, self.make_lease()) bw.remote_write(0, b"a"*25) bw.remote_write(25, b"b"*25) bw.remote_write(50, b"c"*7) # last block may be short @@ -158,8 +156,7 @@ class Bucket(unittest.TestCase): incoming, final = self.make_workdir( "test_write_past_size_errors-{}".format(i) ) - bw = BucketWriter(self, incoming, final, 200, self.make_lease(), - FakeCanary()) + bw = BucketWriter(self, incoming, final, 200, self.make_lease()) with self.assertRaises(DataTooLargeError): bw.remote_write(offset, b"a" * length) @@ -179,7 +176,6 @@ class Bucket(unittest.TestCase): incoming, final = self.make_workdir("overlapping_writes_{}".format(uuid4())) bw = BucketWriter( self, incoming, final, length, self.make_lease(), - FakeCanary() ) # Three writes: 10-19, 30-39, 50-59. This allows for a bunch of holes. bw.remote_write(10, expected_data[10:20]) @@ -218,7 +214,6 @@ class Bucket(unittest.TestCase): incoming, final = self.make_workdir("overlapping_writes_{}".format(uuid4())) bw = BucketWriter( self, incoming, final, length, self.make_lease(), - FakeCanary() ) # Three writes: 10-19, 30-39, 50-59. This allows for a bunch of holes. bw.remote_write(10, b"1" * 10) @@ -318,8 +313,7 @@ class BucketProxy(unittest.TestCase): final = os.path.join(basedir, "bucket") fileutil.make_dirs(basedir) fileutil.make_dirs(os.path.join(basedir, "tmp")) - bw = BucketWriter(self, incoming, final, size, self.make_lease(), - FakeCanary()) + bw = BucketWriter(self, incoming, final, size, self.make_lease()) rb = RemoteBucket(bw) return bw, rb, final @@ -669,7 +663,7 @@ class Server(unittest.TestCase): # the size we request. OVERHEAD = 3*4 LEASE_SIZE = 4+32+32+4 - canary = FakeCanary(True) + canary = FakeCanary() already, writers = self.allocate(ss, b"vid1", [0,1,2], 1000, canary) self.failUnlessEqual(len(writers), 3) # now the StorageServer should have 3000 bytes provisionally @@ -677,16 +671,14 @@ class Server(unittest.TestCase): self.failUnlessEqual(len(ss._active_writers), 3) # allocating 1001-byte shares only leaves room for one - already2, writers2 = self.allocate(ss, b"vid2", [0,1,2], 1001, canary) + canary2 = FakeCanary() + already2, writers2 = self.allocate(ss, b"vid2", [0,1,2], 1001, canary2) self.failUnlessEqual(len(writers2), 1) self.failUnlessEqual(len(ss._active_writers), 4) # we abandon the first set, so their provisional allocation should be # returned - - del already - del writers - gc.collect() + canary.disconnected() self.failUnlessEqual(len(ss._active_writers), 1) # now we have a provisional allocation of 1001 bytes @@ -697,9 +689,6 @@ class Server(unittest.TestCase): for bw in writers2.values(): bw.remote_write(0, b"a"*25) bw.remote_close() - del already2 - del writers2 - del bw self.failUnlessEqual(len(ss._active_writers), 0) # this also changes the amount reported as available by call_get_disk_stats @@ -707,13 +696,12 @@ class Server(unittest.TestCase): # now there should be ALLOCATED=1001+12+72=1085 bytes allocated, and # 5000-1085=3915 free, therefore we can fit 39 100byte shares - already3, writers3 = self.allocate(ss, b"vid3", list(range(100)), 100, canary) + canary3 = FakeCanary() + already3, writers3 = self.allocate(ss, b"vid3", list(range(100)), 100, canary3) self.failUnlessEqual(len(writers3), 39) self.failUnlessEqual(len(ss._active_writers), 39) - del already3 - del writers3 - gc.collect() + canary3.disconnected() self.failUnlessEqual(len(ss._active_writers), 0) ss.disownServiceParent() From 58d7e2f62785eaa698107528a06469e86f1cbf05 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 29 Sep 2021 13:58:53 -0400 Subject: [PATCH 246/269] Simplify implementation. --- src/allmydata/storage/immutable.py | 11 ----------- src/allmydata/storage/server.py | 28 +++++++++++++--------------- src/allmydata/test/test_storage.py | 12 ++++++------ 3 files changed, 19 insertions(+), 32 deletions(-) diff --git a/src/allmydata/storage/immutable.py b/src/allmydata/storage/immutable.py index d3b6ce875..b8b18f140 100644 --- a/src/allmydata/storage/immutable.py +++ b/src/allmydata/storage/immutable.py @@ -204,17 +204,6 @@ class ShareFile(object): self.unlink() return space_freed -# TODOs -# Batch 1: -# - bucketwriter dict in the server, to persist them + TEST of persistence -# - aborting bucketwriter removes it from server persistent + TEST -# - disconnect still aborts _for Foolscap only_ -# - existing in-use buckets are not returned _for Foolscap only_ -# - this implies splitting remote_allocate_buckets into generic and Foolscap-y parts -# Batch 2: -# - scheduled events for aborting bucketwriter + TEST -# - bucketwriter writes delay cancellation + TEST - @implementer(RIBucketWriter) class BucketWriter(Referenceable): # type: ignore # warner/foolscap#78 diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py index 92b3d2a1b..2a4b9a54d 100644 --- a/src/allmydata/storage/server.py +++ b/src/allmydata/storage/server.py @@ -121,12 +121,14 @@ class StorageServer(service.MultiService, Referenceable): self.lease_checker.setServiceParent(self) self._get_current_time = get_current_time - # Currently being-written Bucketwriters. TODO Can probably refactor so - # active_writers is unnecessary, do as second pass. - # Map BucketWriter -> (storage_index, share_num) - self._active_writers = {} # type: Dict[BucketWriter, (bytes,int)] - # Map (storage_index, share_num) -> BucketWriter: - self._bucket_writers = {} # type: Dict[(bytes, int),BucketWriter] + # Currently being-written Bucketwriters. For Foolscap, lifetime is tied + # to connection: when disconnection happens, the BucketWriters are + # removed. For HTTP, this makes no sense, so there will be + # timeout-based cleanup; see + # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3807. + + # Map in-progress filesystem path -> BucketWriter: + self._bucket_writers = {} # type: Dict[str,BucketWriter] # Canaries and disconnect markers for BucketWriters created via Foolscap: self._bucket_writer_disconnect_markers = {} # type: Dict[BucketWriter,(IRemoteReference, object)] @@ -247,7 +249,7 @@ class StorageServer(service.MultiService, Referenceable): def allocated_size(self): space = 0 - for bw in self._active_writers: + for bw in self._bucket_writers.values(): space += bw.allocated_size() return space @@ -275,7 +277,6 @@ class StorageServer(service.MultiService, Referenceable): def allocate_buckets(self, storage_index, renew_secret, cancel_secret, sharenums, allocated_size, - include_in_progress, owner_num=0): """ Generic bucket allocation API. @@ -328,8 +329,7 @@ class StorageServer(service.MultiService, Referenceable): # great! we already have it. easy. pass elif os.path.exists(incominghome): - # TODO use include_in_progress - # Note that we don't create BucketWriters for shnums that + # For Foolscap we don't create BucketWriters for shnums that # have a partial share (in incoming/), so if a second upload # occurs while the first is still in progress, the second # uploader will use different storage servers. @@ -341,8 +341,7 @@ class StorageServer(service.MultiService, Referenceable): if self.no_storage: bw.throw_out_all_data = True bucketwriters[shnum] = bw - self._active_writers[bw] = (storage_index, shnum) - self._bucket_writers[(storage_index, shnum)] = bw + self._bucket_writers[incominghome] = bw if limited: remaining_space -= max_space_per_bucket else: @@ -362,7 +361,7 @@ class StorageServer(service.MultiService, Referenceable): """Foolscap-specific ``allocate_buckets()`` API.""" alreadygot, bucketwriters = self.allocate_buckets( storage_index, renew_secret, cancel_secret, sharenums, allocated_size, - include_in_progress=False, owner_num=owner_num, + owner_num=owner_num, ) # Abort BucketWriters if disconnection happens. for bw in bucketwriters.values(): @@ -413,8 +412,7 @@ class StorageServer(service.MultiService, Referenceable): def bucket_writer_closed(self, bw, consumed_size): if self.stats_provider: self.stats_provider.count('storage_server.bytes_added', consumed_size) - storage_index, shnum = self._active_writers.pop(bw) - del self._bucket_writers[(storage_index, shnum)] + del self._bucket_writers[bw.incominghome] if bw in self._bucket_writer_disconnect_markers: canary, disconnect_marker = self._bucket_writer_disconnect_markers.pop(bw) canary.dontNotifyOnDisconnect(disconnect_marker) diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index 3b36f293f..640280bc3 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -668,19 +668,19 @@ class Server(unittest.TestCase): self.failUnlessEqual(len(writers), 3) # now the StorageServer should have 3000 bytes provisionally # allocated, allowing only 2000 more to be claimed - self.failUnlessEqual(len(ss._active_writers), 3) + self.failUnlessEqual(len(ss._bucket_writers), 3) # allocating 1001-byte shares only leaves room for one canary2 = FakeCanary() already2, writers2 = self.allocate(ss, b"vid2", [0,1,2], 1001, canary2) self.failUnlessEqual(len(writers2), 1) - self.failUnlessEqual(len(ss._active_writers), 4) + self.failUnlessEqual(len(ss._bucket_writers), 4) # we abandon the first set, so their provisional allocation should be # returned canary.disconnected() - self.failUnlessEqual(len(ss._active_writers), 1) + self.failUnlessEqual(len(ss._bucket_writers), 1) # now we have a provisional allocation of 1001 bytes # and we close the second set, so their provisional allocation should @@ -689,7 +689,7 @@ class Server(unittest.TestCase): for bw in writers2.values(): bw.remote_write(0, b"a"*25) bw.remote_close() - self.failUnlessEqual(len(ss._active_writers), 0) + self.failUnlessEqual(len(ss._bucket_writers), 0) # this also changes the amount reported as available by call_get_disk_stats allocated = 1001 + OVERHEAD + LEASE_SIZE @@ -699,11 +699,11 @@ class Server(unittest.TestCase): canary3 = FakeCanary() already3, writers3 = self.allocate(ss, b"vid3", list(range(100)), 100, canary3) self.failUnlessEqual(len(writers3), 39) - self.failUnlessEqual(len(ss._active_writers), 39) + self.failUnlessEqual(len(ss._bucket_writers), 39) canary3.disconnected() - self.failUnlessEqual(len(ss._active_writers), 0) + self.failUnlessEqual(len(ss._bucket_writers), 0) ss.disownServiceParent() del ss From f8604e239406278dc7082ddfd730524b035f3800 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 29 Sep 2021 14:00:11 -0400 Subject: [PATCH 247/269] Fix flakes. --- src/allmydata/test/test_istorageserver.py | 2 -- src/allmydata/test/test_storage.py | 1 - 2 files changed, 3 deletions(-) diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index 25b814237..9ad6a8224 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -21,8 +21,6 @@ if PY2: from random import Random from twisted.internet.defer import inlineCallbacks -from twisted.internet import reactor -from twisted.internet.task import deferLater from foolscap.api import Referenceable, RemoteException diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index 640280bc3..d18960a1e 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -19,7 +19,6 @@ import platform import stat import struct import shutil -import gc from uuid import uuid4 from twisted.trial import unittest From 016d6b4530af011c5ff470846d82d85a941848c3 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 29 Sep 2021 14:10:14 -0400 Subject: [PATCH 248/269] Fix spurious type checking error. --- src/allmydata/storage/server.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py index 2a4b9a54d..041783a4e 100644 --- a/src/allmydata/storage/server.py +++ b/src/allmydata/storage/server.py @@ -274,10 +274,10 @@ class StorageServer(service.MultiService, Referenceable): } return version - def allocate_buckets(self, storage_index, - renew_secret, cancel_secret, - sharenums, allocated_size, - owner_num=0): + def _allocate_buckets(self, storage_index, + renew_secret, cancel_secret, + sharenums, allocated_size, + owner_num=0): """ Generic bucket allocation API. """ @@ -359,7 +359,7 @@ class StorageServer(service.MultiService, Referenceable): sharenums, allocated_size, canary, owner_num=0): """Foolscap-specific ``allocate_buckets()`` API.""" - alreadygot, bucketwriters = self.allocate_buckets( + alreadygot, bucketwriters = self._allocate_buckets( storage_index, renew_secret, cancel_secret, sharenums, allocated_size, owner_num=owner_num, ) From 23fd11be43b2978e07d9be2c0685be0469f47149 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 29 Sep 2021 14:13:18 -0400 Subject: [PATCH 249/269] Expand explanation. --- src/allmydata/test/test_istorageserver.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index 9ad6a8224..29ce272e2 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -137,6 +137,10 @@ class IStorageServerImmutableAPIsTestsMixin(object): (In the real world one shouldn't do that, but writing different data is a good way to test that the original data really was wiped.) + + HTTP protocol should skip this test, since disconnection is meaningless + concept; this is more about testing implicit contract the Foolscap + implementation depends on doesn't change as we refactor things. """ storage_index, renew_secret, cancel_secret = ( new_storage_index(), From 1f6daf02eb05d48f4ddce8f3443706473dec061b Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 29 Sep 2021 15:15:56 -0400 Subject: [PATCH 250/269] news fragment --- newsfragments/3808.installation | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/3808.installation diff --git a/newsfragments/3808.installation b/newsfragments/3808.installation new file mode 100644 index 000000000..157f08a0c --- /dev/null +++ b/newsfragments/3808.installation @@ -0,0 +1 @@ +Tahoe-LAFS now supports running on NixOS 21.05 with Python 3. From fc01835a56ee260970887eb999ed4d4dae9c7b4d Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 29 Sep 2021 15:16:01 -0400 Subject: [PATCH 251/269] ci configuration --- .circleci/config.yml | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 62d1bd752..ea3d50de5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -42,6 +42,9 @@ workflows: - "nixos-19-09": {} + - "nixos-21-05": + {} + # Test against PyPy 2.7 - "pypy27-buster": {} @@ -438,8 +441,7 @@ jobs: image: "tahoelafsci/fedora:29-py" user: "nobody" - - nixos-19-09: + nixos-19-09: &NIXOS docker: # Run in a highly Nix-capable environment. - <<: *DOCKERHUB_AUTH @@ -465,6 +467,15 @@ jobs: # them in parallel. nix-build --cores 3 --max-jobs 2 nix/ + nixos-21-05: + <<: *NIXOS + + environment: + # Note this doesn't look more similar to the 19.09 NIX_PATH URL because + # there was some internal shuffling by the NixOS project about how they + # publish stable revisions. + NIX_PATH: "nixpkgs=https://github.com/NixOS/nixpkgs/archive/refs/heads/nixos-21.05-small.tar.gz" + typechecks: docker: - <<: *DOCKERHUB_AUTH From 49ee4b8acfff3820f80a01d0fb8d2eab60f0a1d5 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 29 Sep 2021 15:27:17 -0400 Subject: [PATCH 252/269] callPackage not directly available from python-self in newer nixpkgs --- nix/overlays.nix | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/nix/overlays.nix b/nix/overlays.nix index 14a47ca5a..f5ef72cc0 100644 --- a/nix/overlays.nix +++ b/nix/overlays.nix @@ -2,25 +2,25 @@ self: super: { python27 = super.python27.override { packageOverrides = python-self: python-super: { # eliot is not part of nixpkgs at all at this time. - eliot = python-self.callPackage ./eliot.nix { }; + eliot = python-self.pythonPackages.callPackage ./eliot.nix { }; # NixOS autobahn package has trollius as a dependency, although # it is optional. Trollius is unmaintained and fails on CI. - autobahn = python-super.callPackage ./autobahn.nix { }; + autobahn = python-super.pythonPackages.callPackage ./autobahn.nix { }; # Porting to Python 3 is greatly aided by the future package. A # slightly newer version than appears in nixos 19.09 is helpful. - future = python-super.callPackage ./future.nix { }; + future = python-super.pythonPackages.callPackage ./future.nix { }; # Need version of pyutil that supports Python 3. The version in 19.09 # is too old. - pyutil = python-super.callPackage ./pyutil.nix { }; + pyutil = python-super.pythonPackages.callPackage ./pyutil.nix { }; # Need a newer version of Twisted, too. - twisted = python-super.callPackage ./twisted.nix { }; + twisted = python-super.pythonPackages.callPackage ./twisted.nix { }; # collections-extended is not part of nixpkgs at this time. - collections-extended = python-super.callPackage ./collections-extended.nix { }; + collections-extended = python-super.pythonPackages.callPackage ./collections-extended.nix { }; }; }; } From 5a3028bdabf1a4674da8576d78f0026fb4f37517 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 29 Sep 2021 15:46:18 -0400 Subject: [PATCH 253/269] add a python3 expression most deps are in nixpkgs now but we still need an overlay for th very very recent collections-extended dependency --- .circleci/config.yml | 6 ++++-- nix/overlays.nix | 7 +++++++ nix/py3.nix | 7 +++++++ 3 files changed, 18 insertions(+), 2 deletions(-) create mode 100644 nix/py3.nix diff --git a/.circleci/config.yml b/.circleci/config.yml index ea3d50de5..dd877ca7f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -449,6 +449,7 @@ jobs: environment: NIX_PATH: "nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/nixos-19.09-small.tar.gz" + SOURCE: "nix/" steps: - "checkout" @@ -465,7 +466,7 @@ jobs: # build a couple simple little dependencies that don't take # advantage of multiple cores and we get a little speedup by doing # them in parallel. - nix-build --cores 3 --max-jobs 2 nix/ + nix-build --cores 3 --max-jobs 2 "$SOURCE" nixos-21-05: <<: *NIXOS @@ -474,7 +475,8 @@ jobs: # Note this doesn't look more similar to the 19.09 NIX_PATH URL because # there was some internal shuffling by the NixOS project about how they # publish stable revisions. - NIX_PATH: "nixpkgs=https://github.com/NixOS/nixpkgs/archive/refs/heads/nixos-21.05-small.tar.gz" + NIX_PATH: "nixpkgs=https://github.com/NixOS/nixpkgs/archive/refs/d32b07e6df276d78e3640eb43882b80c9b2b3459.tar.gz" + SOURCE: "nix/py3.nix" typechecks: docker: diff --git a/nix/overlays.nix b/nix/overlays.nix index f5ef72cc0..fbd0ce3bb 100644 --- a/nix/overlays.nix +++ b/nix/overlays.nix @@ -23,4 +23,11 @@ self: super: { collections-extended = python-super.pythonPackages.callPackage ./collections-extended.nix { }; }; }; + + python39 = super.python39.override { + packageOverrides = python-self: python-super: { + # collections-extended is not part of nixpkgs at this time. + collections-extended = python-super.pythonPackages.callPackage ./collections-extended.nix { }; + }; + }; } diff --git a/nix/py3.nix b/nix/py3.nix new file mode 100644 index 000000000..34ede49dd --- /dev/null +++ b/nix/py3.nix @@ -0,0 +1,7 @@ +# This is the main entrypoint for the Tahoe-LAFS derivation. +{ pkgs ? import { } }: +# Add our Python packages to nixpkgs to simplify the expression for the +# Tahoe-LAFS derivation. +let pkgs' = pkgs.extend (import ./overlays.nix); +# Evaluate the expression for our Tahoe-LAFS derivation. +in pkgs'.python39.pkgs.callPackage ./tahoe-lafs.nix { } From 49df402f0762b34c88b01d183b9d217da117cc79 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Wed, 29 Sep 2021 15:48:33 -0400 Subject: [PATCH 254/269] maybe this is the right url --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index dd877ca7f..2fc8e88e7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -475,7 +475,7 @@ jobs: # Note this doesn't look more similar to the 19.09 NIX_PATH URL because # there was some internal shuffling by the NixOS project about how they # publish stable revisions. - NIX_PATH: "nixpkgs=https://github.com/NixOS/nixpkgs/archive/refs/d32b07e6df276d78e3640eb43882b80c9b2b3459.tar.gz" + NIX_PATH: "nixpkgs=https://github.com/NixOS/nixpkgs/archive/d32b07e6df276d78e3640eb43882b80c9b2b3459.tar.gz" SOURCE: "nix/py3.nix" typechecks: From 17a670dfb5b45a1bf62049941f4b964ac9411443 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 4 Oct 2021 10:37:37 -0400 Subject: [PATCH 255/269] Test for aborting bucket upload. --- newsfragments/3798.minor | 0 src/allmydata/test/test_istorageserver.py | 41 +++++++++++++++++++++++ 2 files changed, 41 insertions(+) create mode 100644 newsfragments/3798.minor diff --git a/newsfragments/3798.minor b/newsfragments/3798.minor new file mode 100644 index 000000000..e69de29bb diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index a3ea27892..554cc6a29 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -313,6 +313,47 @@ class IStorageServerImmutableAPIsTestsMixin(object): self.assertEqual((yield buckets[0].callRemote("read", 0, 25)), b"1" * 25) + @inlineCallbacks + def test_abort(self): + """ + If we call ``abort`` on the ``RIBucketWriter`` disconnect in the middle + of writing to a bucket, all data is wiped, and it's even possible to + write different data to the bucket. + + (In the real world one probably wouldn't do that, but writing different + data is a good way to test that the original data really was wiped.) + """ + storage_index, renew_secret, cancel_secret = ( + new_storage_index(), + new_secret(), + new_secret(), + ) + (_, allocated) = yield self.storage_server.allocate_buckets( + storage_index, + renew_secret, + cancel_secret, + sharenums={0}, + allocated_size=1024, + canary=Referenceable(), + ) + + # Bucket 0 is fully written in one go. + yield allocated[0].callRemote("write", 0, b"1" * 1024) + + # Abort the upload: + yield allocated[0].callRemote("abort") + + # Write different data with no complaint: + (_, allocated) = yield self.storage_server.allocate_buckets( + storage_index, + renew_secret, + cancel_secret, + sharenums={0}, + allocated_size=1024, + canary=Referenceable(), + ) + yield allocated[0].callRemote("write", 0, b"2" * 1024) + @inlineCallbacks def test_get_buckets_skips_unfinished_buckets(self): """ From add34efffbb4c1bc0193cfaae9ea6c28c5c93762 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 4 Oct 2021 10:58:42 -0400 Subject: [PATCH 256/269] News file. --- newsfragments/3810.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3810.minor diff --git a/newsfragments/3810.minor b/newsfragments/3810.minor new file mode 100644 index 000000000..e69de29bb From 2b83edc5b30f910eaafaf6831f26b3d8b71b27a9 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 4 Oct 2021 11:00:16 -0400 Subject: [PATCH 257/269] Use macos-10.15 for Python 2.7. --- .github/workflows/ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e161ec243..45b2986a3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -28,7 +28,7 @@ jobs: - 3.9 include: # On macOS don't bother with 3.6-3.8, just to get faster builds. - - os: macos-latest + - os: macos-10.15 python-version: 2.7 - os: macos-latest python-version: 3.9 @@ -168,7 +168,7 @@ jobs: - 3.9 include: # On macOS don't bother with 3.6, just to get faster builds. - - os: macos-latest + - os: macos-10.15 python-version: 2.7 - os: macos-latest python-version: 3.9 @@ -183,7 +183,7 @@ jobs: # We have to use an older version of Tor for running integration # tests on macOS. - name: Install Tor [macOS, ${{ matrix.python-version }} ] - if: ${{ matrix.os == 'macos-latest' }} + if: ${{ contains(matrix.os, 'macos') }} run: | brew extract --version 0.4.5.8 tor homebrew/cask brew install tor@0.4.5.8 @@ -247,7 +247,7 @@ jobs: fail-fast: false matrix: os: - - macos-latest + - macos-10.15 - windows-latest - ubuntu-latest python-version: From aef581628f356ae2a8302b1d3b18eb1b72468901 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 6 Oct 2021 15:10:58 -0400 Subject: [PATCH 258/269] Add discussion. --- docs/proposed/http-storage-node-protocol.rst | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/proposed/http-storage-node-protocol.rst index e3bd48128..bb5dc28ad 100644 --- a/docs/proposed/http-storage-node-protocol.rst +++ b/docs/proposed/http-storage-node-protocol.rst @@ -515,6 +515,24 @@ Clients should upload chunks in re-assembly order. } +Discussion +`````````` + +``PUT`` verbs are only supposed to be used to replace the whole resource, +thus the use of ``PATCH``. +From RFC 7231:: + + An origin server that allows PUT on a given target resource MUST send + a 400 (Bad Request) response to a PUT request that contains a + Content-Range header field (Section 4.2 of [RFC7233]), since the + payload is likely to be partial content that has been mistakenly PUT + as a full representation. Partial content updates are possible by + targeting a separately identified resource with state that overlaps a + portion of the larger resource, or by using a different method that + has been specifically defined for partial updates (for example, the + PATCH method defined in [RFC5789]). + + ``POST /v1/immutable/:storage_index/:share_number/corrupt`` !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! From 82cbce6b7e258a651ff2e8de580fc10f5b0e9468 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 6 Oct 2021 15:12:22 -0400 Subject: [PATCH 259/269] Better explanation. --- src/allmydata/test/test_istorageserver.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index 931cf6ed6..26d976d00 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -647,7 +647,9 @@ class IStorageServerMutableAPIsTestsMixin(object): tw_vectors={ 0: ([], [(0, b"abcdefg")], 7), 1: ([], [(0, b"0123"), (4, b"456")], 7), - 2: ([], [(0, b"0123")], 4), + # This will never get read from, just here to show we only read + # from shares explicitly requested by slot_readv: + 2: ([], [(0, b"XYZW")], 4), }, r_vector=[], ) From bf176144c5f0fc99fa25f7c250927a3fe9d43b50 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 6 Oct 2021 15:18:00 -0400 Subject: [PATCH 260/269] Handle double-disconnect, should it happen by mistake. --- src/allmydata/test/common_util.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/allmydata/test/common_util.py b/src/allmydata/test/common_util.py index de8d774b3..d2d20916d 100644 --- a/src/allmydata/test/common_util.py +++ b/src/allmydata/test/common_util.py @@ -314,10 +314,15 @@ class FakeCanary(object): def getPeer(self): return "" - # For use by tests: def disconnected(self): - for (f, args, kwargs) in list(self.disconnectors.values()): - f(*args, **kwargs) + """Disconnect the canary, to be called by test code. + + Can only happen once. + """ + if self.disconnectors is not None: + for (f, args, kwargs) in list(self.disconnectors.values()): + f(*args, **kwargs) + self.disconnectors = None class ShouldFailMixin(object): From 807363adc9ef94035c4c167ed84fb47bce690fbf Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 6 Oct 2021 16:41:50 -0400 Subject: [PATCH 261/269] Reduce duplication. --- src/allmydata/test/test_istorageserver.py | 67 +++++++++-------------- 1 file changed, 26 insertions(+), 41 deletions(-) diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index ee640bdda..40dcdc8bb 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -130,17 +130,16 @@ class IStorageServerImmutableAPIsTestsMixin(object): self.assertEqual(set(allocated2.keys()), {4}) @inlineCallbacks - def test_disconnection(self): + def abort_or_disconnect_half_way(self, abort_or_disconnect): """ - If we disconnect in the middle of writing to a bucket, all data is - wiped, and it's even possible to write different data to the bucket. + If we disconnect/abort in the middle of writing to a bucket, all data + is wiped, and it's even possible to write different data to the bucket. (In the real world one shouldn't do that, but writing different data is a good way to test that the original data really was wiped.) - HTTP protocol should skip this test, since disconnection is meaningless - concept; this is more about testing implicit contract the Foolscap - implementation depends on doesn't change as we refactor things. + ``abort_or_disconnect`` is a callback that takes a bucket and aborts up + load, or perhaps disconnects the whole connection. """ storage_index, renew_secret, cancel_secret = ( new_storage_index(), @@ -159,8 +158,8 @@ class IStorageServerImmutableAPIsTestsMixin(object): # Bucket 1 is fully written in one go. yield allocated[0].callRemote("write", 0, b"1" * 1024) - # Disconnect: - yield self.disconnect() + # Disconnect or abort, depending on the test: + yield abort_or_disconnect(allocated[0]) # Write different data with no complaint: (_, allocated) = yield self.storage_server.allocate_buckets( @@ -173,6 +172,20 @@ class IStorageServerImmutableAPIsTestsMixin(object): ) yield allocated[0].callRemote("write", 0, b"2" * 1024) + def test_disconnection(self): + """ + If we disconnect in the middle of writing to a bucket, all data is + wiped, and it's even possible to write different data to the bucket. + + (In the real world one shouldn't do that, but writing different data is + a good way to test that the original data really was wiped.) + + HTTP protocol should skip this test, since disconnection is meaningless + concept; this is more about testing implicit contract the Foolscap + implementation depends on doesn't change as we refactor things. + """ + return self.abort_or_disconnect_half_way(lambda _: self.disconnect()) + @inlineCallbacks def test_written_shares_are_allocated(self): """ @@ -313,46 +326,18 @@ class IStorageServerImmutableAPIsTestsMixin(object): self.assertEqual((yield buckets[0].callRemote("read", 0, 25)), b"1" * 25) - @inlineCallbacks def test_abort(self): """ - If we call ``abort`` on the ``RIBucketWriter`` disconnect in the middle - of writing to a bucket, all data is wiped, and it's even possible to - write different data to the bucket. + If we call ``abort`` on the ``RIBucketWriter`` to disconnect in the + middle of writing to a bucket, all data is wiped, and it's even + possible to write different data to the bucket. (In the real world one probably wouldn't do that, but writing different data is a good way to test that the original data really was wiped.) """ - storage_index, renew_secret, cancel_secret = ( - new_storage_index(), - new_secret(), - new_secret(), + return self.abort_or_disconnect_half_way( + lambda bucket: bucket.callRemote("abort") ) - (_, allocated) = yield self.storage_server.allocate_buckets( - storage_index, - renew_secret, - cancel_secret, - sharenums={0}, - allocated_size=1024, - canary=Referenceable(), - ) - - # Bucket 0 is fully written in one go. - yield allocated[0].callRemote("write", 0, b"1" * 1024) - - # Abort the upload: - yield allocated[0].callRemote("abort") - - # Write different data with no complaint: - (_, allocated) = yield self.storage_server.allocate_buckets( - storage_index, - renew_secret, - cancel_secret, - sharenums={0}, - allocated_size=1024, - canary=Referenceable(), - ) - yield allocated[0].callRemote("write", 0, b"2" * 1024) @inlineCallbacks def test_get_buckets_skips_unfinished_buckets(self): From 984b4ac45e89c3f53b5325006e0d33b15165ab26 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 13 Oct 2021 13:56:14 -0400 Subject: [PATCH 262/269] News file. --- newsfragments/3812.minor | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 newsfragments/3812.minor diff --git a/newsfragments/3812.minor b/newsfragments/3812.minor new file mode 100644 index 000000000..e69de29bb From 68e8e0a7d5b88568bd01c1a14957f972830d1f54 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 14 Oct 2021 10:58:41 -0400 Subject: [PATCH 263/269] a stab at using setup.cfg and setuptools_scm --- CLASSIFIERS.txt | 29 +++ towncrier.toml => pyproject.toml | 10 + setup.cfg | 215 +++++++++++++++- setup.py | 429 +------------------------------ src/allmydata/__init__.py | 40 ++- tox.ini | 8 +- 6 files changed, 269 insertions(+), 462 deletions(-) create mode 100644 CLASSIFIERS.txt rename towncrier.toml => pyproject.toml (80%) diff --git a/CLASSIFIERS.txt b/CLASSIFIERS.txt new file mode 100644 index 000000000..7aa2e35b1 --- /dev/null +++ b/CLASSIFIERS.txt @@ -0,0 +1,29 @@ +Development Status :: 5 - Production/Stable +Environment :: Console +Environment :: Web Environment +License :: OSI Approved :: GNU General Public License (GPL) +License :: DFSG approved +License :: Other/Proprietary License +Intended Audience :: Developers +Intended Audience :: End Users/Desktop +Intended Audience :: System Administrators +Operating System :: Microsoft +Operating System :: Microsoft :: Windows +Operating System :: Unix +Operating System :: POSIX :: Linux +Operating System :: POSIX +Operating System :: MacOS :: MacOS X +Operating System :: OS Independent +Natural Language :: English +Programming Language :: C +Programming Language :: Python +Programming Language :: Python :: 2 +Programming Language :: Python :: 2.7 +Topic :: Utilities +Topic :: System :: Systems Administration +Topic :: System :: Filesystems +Topic :: System :: Distributed Computing +Topic :: Software Development :: Libraries +Topic :: System :: Archiving :: Backup +Topic :: System :: Archiving :: Mirroring +Topic :: System :: Archiving diff --git a/towncrier.toml b/pyproject.toml similarity index 80% rename from towncrier.toml rename to pyproject.toml index b8b561a98..7c97001aa 100644 --- a/towncrier.toml +++ b/pyproject.toml @@ -1,3 +1,13 @@ +# https://setuptools.pypa.io/en/latest/build_meta.html +# https://github.com/pypa/setuptools_scm +[build-system] +requires = ["setuptools>=45", "wheel", "setuptools_scm>=6.2"] +build-backend = "setuptools.build_meta" + +[tool.setuptools_scm] +write_to = "src/allmydata/_version.py" +tag_regex = "^tahoe-lafs-(?P[vV]?\\d+(?:\\.\\d+){0,2}[^\\+]*)(?:\\+.*)?$" + [tool.towncrier] package_dir = "src" package = "allmydata" diff --git a/setup.cfg b/setup.cfg index f4539279e..21e1fb663 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,10 +1,211 @@ -[aliases] -build = update_version build -sdist = update_version sdist -install = update_version install -develop = update_version develop -bdist_egg = update_version bdist_egg -bdist_wheel = update_version bdist_wheel +# https://setuptools.pypa.io/en/latest/userguide/declarative_config.html#declarative-config +[metadata] +name = tahoe-lafs + +[options] +version = attr:allmydata._version.version +description = secure, decentralized, fault-tolerant file store +long_description = file: README.rst +author = the Tahoe-LAFS project +author_email = tahoe-dev@lists.tahoe-lafs.org +project_urls = + website=https://tahoe-lafs.org/ + documentation=https://tahoe-lafs.readthedocs.org/ + source=https://github.com/tahoe-lafs/tahoe-lafs +classifiers = file: CLASSIFIERS.txt +license_files = COPYING.GPL, COPYING.TGPPL.rst +# tell setuptools to find our package source files automatically +packages = find: +# find packages beneath the src directory +package_dir= + =src +include_package_data = True + +# We support Python 2.7 and many newer versions of Python 3. +python_requires = >=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.* + +install_requires = + # we don't need much out of setuptools but the version checking stuff + # needs pkg_resources and PEP 440 version specifiers. + setuptools >= 28.8.0 + + zfec >= 1.1.0 + + # zope.interface >= 3.6.0 is required for Twisted >= 12.1.0. + zope.interface >= 3.6.0 + + # * foolscap < 0.5.1 had a performance bug which spent O(N**2) CPU for + # transferring large mutable files of size N. + # * foolscap < 0.6 is incompatible with Twisted 10.2.0. + # * foolscap 0.6.1 quiets a DeprecationWarning. + # * foolscap < 0.6.3 is incompatible with Twisted 11.1.0 and newer. + # * foolscap 0.8.0 generates 2048-bit RSA-with-SHA-256 signatures, + # rather than 1024-bit RSA-with-MD5. This also allows us to work + # with a FIPS build of OpenSSL. + # * foolscap >= 0.12.3 provides tcp/tor/i2p connection handlers we need, + # and allocate_tcp_port + # * foolscap >= 0.12.5 has ConnectionInfo and ReconnectionInfo + # * foolscap >= 0.12.6 has an i2p.sam_endpoint() that takes kwargs + # * foolscap 0.13.2 drops i2p support completely + # * foolscap >= 21.7 is necessary for Python 3 with i2p support. + foolscap == 0.13.1 ; python_version < '3.0' + foolscap >= 21.7.0 ; python_version > '3.0' + + # * cryptography 2.6 introduced some ed25519 APIs we rely on. Note that + # Twisted[conch] also depends on cryptography and Twisted[tls] + # transitively depends on cryptography. So it's anyone's guess what + # version of cryptography will *really* be installed. + cryptography >= 2.6 + + # * The SFTP frontend depends on Twisted 11.0.0 to fix the SSH server + # rekeying bug + # * The SFTP frontend and manhole depend on the conch extra. However, we + # can't explicitly declare that without an undesirable dependency on gmpy, + # as explained in ticket #2740. + # * Due to a setuptools bug, we need to declare a dependency on the tls + # extra even though we only depend on it via foolscap. + # * Twisted >= 15.1.0 is the first version that provided the [tls] extra. + # * Twisted-16.1.0 fixes https://twistedmatrix.com/trac/ticket/8223, + # which otherwise causes test_system to fail (DirtyReactorError, due to + # leftover timers) + # * Twisted-16.4.0 introduces `python -m twisted.trial` which is needed + # for coverage testing + # * Twisted 16.6.0 drops the undesirable gmpy dependency from the conch + # extra, letting us use that extra instead of trying to duplicate its + # dependencies here. Twisted[conch] >18.7 introduces a dependency on + # bcrypt. It is nice to avoid that if the user ends up with an older + # version of Twisted. That's hard to express except by using the extra. + # + # * Twisted 18.4.0 adds `client` and `host` attributes to `Request` in the + # * initializer, needed by logic in our custom `Request` subclass. + # + # In a perfect world, Twisted[conch] would be a dependency of an sftp + # extra. However, pip fails to resolve the dependencies all + # dependencies when asked for Twisted[tls] *and* Twisted[conch]. + # Specifically, Twisted[conch] (as the later requirement) is ignored. + # If there were an Tahoe-LAFS sftp extra that dependended on + # Twisted[conch] and install_requires only included Twisted[tls] then + # `pip install tahoe-lafs[sftp]` would not install requirements + # specified by Twisted[conch]. Since this would be the *whole point* of + # an sftp extra in Tahoe-LAFS, there is no point in having one. + # * Twisted 19.10 introduces Site.getContentFile which we use to get + # temporary upload files placed into a per-node temporary directory. + Twisted[tls,conch] >= 19.10.0 + + PyYAML >= 3.11 + + six >= 1.10.0 + + # for 'tahoe invite' and 'tahoe join' + magic-wormhole >= 0.10.2 + + # Eliot is contemplating dropping Python 2 support. Stick to a version we + # know works on Python 2.7. + eliot ~= 1.7 ; python_version < '3.0' + # On Python 3, we want a new enough version to support custom JSON encoders. + eliot >= 1.13.0 ; python_version > '3.0' + + # Pyrsistent 0.17.0 (which we use by way of Eliot) has dropped + # Python 2 entirely; stick to the version known to work for us. + pyrsistent < 0.17.0 ; python_version < '3.0' + pyrsistent ; python_version > '3.0' + + # A great way to define types of values. + attrs >= 18.2.0 + + # WebSocket library for twisted and asyncio + autobahn >= 19.5.2 + + # Support for Python 3 transition + future >= 0.18.2 + + # Discover local network configuration + netifaces + + # Utility code: + pyutil >= 3.3.0 + + # Linux distribution detection: + distro >= 1.4.0 + + # Backported configparser for Python 2: + configparser ; python_version < '3.0' + + # For the RangeMap datastructure. + collections-extended + + # Duplicate the Twisted pywin32 dependency here. See + # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/2392 for some + # discussion. + pywin32 != 226 ; sys_platform=="win32" + +[options.packages.find] +# inform the setuptools source discovery logic to start in this directory +where = src + +[options.package_data] +allmydata.web = + *.xhtml + static/*.js + static/*.png + static/*.css + static/img/*.png + static/css/*.css + +[options.extras_require] +test = + flake8 + # Pin a specific pyflakes so we don't have different folks + # disagreeing on what is or is not a lint issue. We can bump + # this version from time to time, but we will do it + # intentionally. + pyflakes == 2.2.0 + coverage ~= 5.0 + mock + tox + pytest + pytest-twisted + # XXX: decorator isn't a direct dependency, but pytest-twisted + # depends on decorator, and decorator 5.x isn't compatible with + # Python 2.7. + decorator < 5 + hypothesis >= 3.6.1 + treq + towncrier + testtools + fixtures + beautifulsoup4 + html5lib + junitxml + tenacity + paramiko + pytest-timeout + # Does our OpenMetrics endpoint adhere to the spec: + prometheus-client == 0.11.0 + + # Make sure the tor and i2p tests can run by duplicating the requirements + # for those extras here. + %(tor)s + %(i2p)s + +tor = + # This is exactly what `foolscap[tor]` means but pip resolves the pair of + # dependencies "foolscap[i2p] foolscap[tor]" to "foolscap[i2p]" so we lose + # this if we don't declare it ourselves! + txtorcon >= 0.17.0 + +i2p = + # txi2p has Python 3 support in master branch, but it has not been + # released -- see https://github.com/str4d/txi2p/issues/10. We + # could use a fork for Python 3 until txi2p's maintainers are back + # in action. For Python 2, we could continue using the txi2p + # version about which no one has complained to us so far. + txi2p; python_version < '3.0' + txi2p-tahoe >= 0.3.5; python_version > '3.0' + +[options.entry_points] +console_scripts = + tahoe = allmydata.scripts.runner:run [flake8] # Enforce all pyflakes constraints, and also prohibit tabs for indentation. diff --git a/setup.py b/setup.py index 8c6396937..8bf1ba938 100644 --- a/setup.py +++ b/setup.py @@ -1,427 +1,2 @@ -#! /usr/bin/env python -# -*- coding: utf-8 -*- -import sys - -# Tahoe-LAFS -- secure, distributed storage grid -# -# Copyright © 2006-2012 The Tahoe-LAFS Software Foundation -# -# This file is part of Tahoe-LAFS. -# -# See the docs/about.rst file for licensing information. - -import os, subprocess, re -from io import open - -basedir = os.path.dirname(os.path.abspath(__file__)) - -# locate our version number - -def read_version_py(infname): - try: - verstrline = open(infname, "rt").read() - except EnvironmentError: - return None - else: - VSRE = r"^verstr = ['\"]([^'\"]*)['\"]" - mo = re.search(VSRE, verstrline, re.M) - if mo: - return mo.group(1) - -VERSION_PY_FILENAME = 'src/allmydata/_version.py' -version = read_version_py(VERSION_PY_FILENAME) - -install_requires = [ - # we don't need much out of setuptools but the version checking stuff - # needs pkg_resources and PEP 440 version specifiers. - "setuptools >= 28.8.0", - - "zfec >= 1.1.0", - - # zope.interface >= 3.6.0 is required for Twisted >= 12.1.0. - "zope.interface >= 3.6.0", - - # * foolscap < 0.5.1 had a performance bug which spent O(N**2) CPU for - # transferring large mutable files of size N. - # * foolscap < 0.6 is incompatible with Twisted 10.2.0. - # * foolscap 0.6.1 quiets a DeprecationWarning. - # * foolscap < 0.6.3 is incompatible with Twisted 11.1.0 and newer. - # * foolscap 0.8.0 generates 2048-bit RSA-with-SHA-256 signatures, - # rather than 1024-bit RSA-with-MD5. This also allows us to work - # with a FIPS build of OpenSSL. - # * foolscap >= 0.12.3 provides tcp/tor/i2p connection handlers we need, - # and allocate_tcp_port - # * foolscap >= 0.12.5 has ConnectionInfo and ReconnectionInfo - # * foolscap >= 0.12.6 has an i2p.sam_endpoint() that takes kwargs - # * foolscap 0.13.2 drops i2p support completely - # * foolscap >= 21.7 is necessary for Python 3 with i2p support. - "foolscap == 0.13.1 ; python_version < '3.0'", - "foolscap >= 21.7.0 ; python_version > '3.0'", - - # * cryptography 2.6 introduced some ed25519 APIs we rely on. Note that - # Twisted[conch] also depends on cryptography and Twisted[tls] - # transitively depends on cryptography. So it's anyone's guess what - # version of cryptography will *really* be installed. - "cryptography >= 2.6", - - # * The SFTP frontend depends on Twisted 11.0.0 to fix the SSH server - # rekeying bug - # * The SFTP frontend and manhole depend on the conch extra. However, we - # can't explicitly declare that without an undesirable dependency on gmpy, - # as explained in ticket #2740. - # * Due to a setuptools bug, we need to declare a dependency on the tls - # extra even though we only depend on it via foolscap. - # * Twisted >= 15.1.0 is the first version that provided the [tls] extra. - # * Twisted-16.1.0 fixes https://twistedmatrix.com/trac/ticket/8223, - # which otherwise causes test_system to fail (DirtyReactorError, due to - # leftover timers) - # * Twisted-16.4.0 introduces `python -m twisted.trial` which is needed - # for coverage testing - # * Twisted 16.6.0 drops the undesirable gmpy dependency from the conch - # extra, letting us use that extra instead of trying to duplicate its - # dependencies here. Twisted[conch] >18.7 introduces a dependency on - # bcrypt. It is nice to avoid that if the user ends up with an older - # version of Twisted. That's hard to express except by using the extra. - # - # * Twisted 18.4.0 adds `client` and `host` attributes to `Request` in the - # * initializer, needed by logic in our custom `Request` subclass. - # - # In a perfect world, Twisted[conch] would be a dependency of an "sftp" - # extra. However, pip fails to resolve the dependencies all - # dependencies when asked for Twisted[tls] *and* Twisted[conch]. - # Specifically, "Twisted[conch]" (as the later requirement) is ignored. - # If there were an Tahoe-LAFS sftp extra that dependended on - # Twisted[conch] and install_requires only included Twisted[tls] then - # `pip install tahoe-lafs[sftp]` would not install requirements - # specified by Twisted[conch]. Since this would be the *whole point* of - # an sftp extra in Tahoe-LAFS, there is no point in having one. - # * Twisted 19.10 introduces Site.getContentFile which we use to get - # temporary upload files placed into a per-node temporary directory. - "Twisted[tls,conch] >= 19.10.0", - - "PyYAML >= 3.11", - - "six >= 1.10.0", - - # for 'tahoe invite' and 'tahoe join' - "magic-wormhole >= 0.10.2", - - # Eliot is contemplating dropping Python 2 support. Stick to a version we - # know works on Python 2.7. - "eliot ~= 1.7 ; python_version < '3.0'", - # On Python 3, we want a new enough version to support custom JSON encoders. - "eliot >= 1.13.0 ; python_version > '3.0'", - - # Pyrsistent 0.17.0 (which we use by way of Eliot) has dropped - # Python 2 entirely; stick to the version known to work for us. - "pyrsistent < 0.17.0 ; python_version < '3.0'", - "pyrsistent ; python_version > '3.0'", - - # A great way to define types of values. - "attrs >= 18.2.0", - - # WebSocket library for twisted and asyncio - "autobahn >= 19.5.2", - - # Support for Python 3 transition - "future >= 0.18.2", - - # Discover local network configuration - "netifaces", - - # Utility code: - "pyutil >= 3.3.0", - - # Linux distribution detection: - "distro >= 1.4.0", - - # Backported configparser for Python 2: - "configparser ; python_version < '3.0'", - - # For the RangeMap datastructure. - "collections-extended", -] - -setup_requires = [ - 'setuptools >= 28.8.0', # for PEP-440 style versions -] - -tor_requires = [ - # This is exactly what `foolscap[tor]` means but pip resolves the pair of - # dependencies "foolscap[i2p] foolscap[tor]" to "foolscap[i2p]" so we lose - # this if we don't declare it ourselves! - "txtorcon >= 0.17.0", -] - -i2p_requires = [ - # txi2p has Python 3 support in master branch, but it has not been - # released -- see https://github.com/str4d/txi2p/issues/10. We - # could use a fork for Python 3 until txi2p's maintainers are back - # in action. For Python 2, we could continue using the txi2p - # version about which no one has complained to us so far. - "txi2p; python_version < '3.0'", - "txi2p-tahoe >= 0.3.5; python_version > '3.0'", -] - -if len(sys.argv) > 1 and sys.argv[1] == '--fakedependency': - del sys.argv[1] - install_requires += ["fakedependency >= 1.0.0"] - -from setuptools import find_packages, setup -from setuptools import Command -from setuptools.command import install - - -trove_classifiers=[ - "Development Status :: 5 - Production/Stable", - "Environment :: Console", - "Environment :: Web Environment", - "License :: OSI Approved :: GNU General Public License (GPL)", - "License :: DFSG approved", - "License :: Other/Proprietary License", - "Intended Audience :: Developers", - "Intended Audience :: End Users/Desktop", - "Intended Audience :: System Administrators", - "Operating System :: Microsoft", - "Operating System :: Microsoft :: Windows", - "Operating System :: Unix", - "Operating System :: POSIX :: Linux", - "Operating System :: POSIX", - "Operating System :: MacOS :: MacOS X", - "Operating System :: OS Independent", - "Natural Language :: English", - "Programming Language :: C", - "Programming Language :: Python", - "Programming Language :: Python :: 2", - "Programming Language :: Python :: 2.7", - "Topic :: Utilities", - "Topic :: System :: Systems Administration", - "Topic :: System :: Filesystems", - "Topic :: System :: Distributed Computing", - "Topic :: Software Development :: Libraries", - "Topic :: System :: Archiving :: Backup", - "Topic :: System :: Archiving :: Mirroring", - "Topic :: System :: Archiving", - ] - - -GIT_VERSION_BODY = ''' -# This _version.py is generated from git metadata by the tahoe setup.py. - -__pkgname__ = "%(pkgname)s" -real_version = "%(version)s" -full_version = "%(full)s" -branch = "%(branch)s" -verstr = "%(normalized)s" -__version__ = verstr -''' - -def run_command(args, cwd=None): - use_shell = sys.platform == "win32" - try: - p = subprocess.Popen(args, stdout=subprocess.PIPE, cwd=cwd, shell=use_shell) - except EnvironmentError as e: # if this gives a SyntaxError, note that Tahoe-LAFS requires Python 2.7+ - print("Warning: unable to run %r." % (" ".join(args),)) - print(e) - return None - stdout = p.communicate()[0].strip() - if p.returncode != 0: - print("Warning: %r returned error code %r." % (" ".join(args), p.returncode)) - return None - return stdout - - -def versions_from_git(tag_prefix): - # This runs 'git' from the directory that contains this file. That either - # means someone ran a setup.py command (and this code is in - # versioneer.py, thus the containing directory is the root of the source - # tree), or someone ran a project-specific entry point (and this code is - # in _version.py, thus the containing directory is somewhere deeper in - # the source tree). This only gets called if the git-archive 'subst' - # variables were *not* expanded, and _version.py hasn't already been - # rewritten with a short version string, meaning we're inside a checked - # out source tree. - - # versions_from_git (as copied from python-versioneer) returns strings - # like "1.9.0-25-gb73aba9-dirty", which means we're in a tree with - # uncommited changes (-dirty), the latest checkin is revision b73aba9, - # the most recent tag was 1.9.0, and b73aba9 has 25 commits that weren't - # in 1.9.0 . The narrow-minded NormalizedVersion parser that takes our - # output (meant to enable sorting of version strings) refuses most of - # that. Tahoe uses a function named suggest_normalized_version() that can - # handle "1.9.0.post25", so dumb down our output to match. - - try: - source_dir = os.path.dirname(os.path.abspath(__file__)) - except NameError as e: - # some py2exe/bbfreeze/non-CPython implementations don't do __file__ - print("Warning: unable to find version because we could not obtain the source directory.") - print(e) - return {} - stdout = run_command(["git", "describe", "--tags", "--dirty", "--always"], - cwd=source_dir) - if stdout is None: - # run_command already complained. - return {} - stdout = stdout.decode("ascii") - if not stdout.startswith(tag_prefix): - print("Warning: tag %r doesn't start with prefix %r." % (stdout, tag_prefix)) - return {} - version = stdout[len(tag_prefix):] - pieces = version.split("-") - if len(pieces) == 1: - normalized_version = pieces[0] - else: - normalized_version = "%s.post%s" % (pieces[0], pieces[1]) - - stdout = run_command(["git", "rev-parse", "HEAD"], cwd=source_dir) - if stdout is None: - # run_command already complained. - return {} - full = stdout.decode("ascii").strip() - if version.endswith("-dirty"): - full += "-dirty" - normalized_version += ".dev0" - - # Thanks to Jistanidiot at . - stdout = run_command(["git", "rev-parse", "--abbrev-ref", "HEAD"], cwd=source_dir) - branch = (stdout or b"unknown").decode("ascii").strip() - - # this returns native strings (bytes on py2, unicode on py3) - return {"version": version, "normalized": normalized_version, - "full": full, "branch": branch} - -# setup.cfg has an [aliases] section which runs "update_version" before many -# commands (like "build" and "sdist") that need to know our package version -# ahead of time. If you add different commands (or if we forgot some), you -# may need to add it to setup.cfg and configure it to run update_version -# before your command. - -class UpdateVersion(Command): - description = "update _version.py from revision-control metadata" - user_options = install.install.user_options - - def initialize_options(self): - pass - def finalize_options(self): - pass - def run(self): - global version - verstr = version - if os.path.isdir(os.path.join(basedir, ".git")): - verstr = self.try_from_git() - - if verstr: - self.distribution.metadata.version = verstr - else: - print("""\ -******************************************************************** -Warning: no version information found. This may cause tests to fail. -******************************************************************** -""") - - def try_from_git(self): - # If we change the release tag names, we must change this too - versions = versions_from_git("tahoe-lafs-") - - # setup.py might be run by either py2 or py3 (when run by tox, which - # uses py3 on modern debian/ubuntu distros). We want this generated - # file to contain native strings on both (str=bytes in py2, - # str=unicode in py3) - if versions: - body = GIT_VERSION_BODY % { - "pkgname": self.distribution.get_name(), - "version": versions["version"], - "normalized": versions["normalized"], - "full": versions["full"], - "branch": versions["branch"], - } - f = open(VERSION_PY_FILENAME, "wb") - f.write(body.encode("ascii")) - f.close() - print("Wrote normalized version %r into '%s'" % (versions["normalized"], VERSION_PY_FILENAME)) - - return versions.get("normalized", None) - -class PleaseUseTox(Command): - user_options = [] - def initialize_options(self): - pass - def finalize_options(self): - pass - - def run(self): - print("ERROR: Please use 'tox' to run the test suite.") - sys.exit(1) - -setup_args = {} -if version: - setup_args["version"] = version - -setup(name="tahoe-lafs", # also set in __init__.py - description='secure, decentralized, fault-tolerant file store', - long_description=open('README.rst', 'r', encoding='utf-8').read(), - author='the Tahoe-LAFS project', - author_email='tahoe-dev@lists.tahoe-lafs.org', - url='https://tahoe-lafs.org/', - license='GNU GPL', # see README.rst -- there is an alternative licence - cmdclass={"update_version": UpdateVersion, - "test": PleaseUseTox, - }, - package_dir = {'':'src'}, - packages=find_packages('src') + ['allmydata.test.plugins'], - classifiers=trove_classifiers, - # We support Python 2.7, and we're working on support for 3.6 (the - # highest version that PyPy currently supports). - python_requires=">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*", - install_requires=install_requires, - extras_require={ - # Duplicate the Twisted pywin32 dependency here. See - # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/2392 for some - # discussion. - ':sys_platform=="win32"': ["pywin32 != 226"], - "test": [ - "flake8", - # Pin a specific pyflakes so we don't have different folks - # disagreeing on what is or is not a lint issue. We can bump - # this version from time to time, but we will do it - # intentionally. - "pyflakes == 2.2.0", - "coverage ~= 5.0", - "mock", - "tox", - "pytest", - "pytest-twisted", - # XXX: decorator isn't a direct dependency, but pytest-twisted - # depends on decorator, and decorator 5.x isn't compatible with - # Python 2.7. - "decorator < 5", - "hypothesis >= 3.6.1", - "treq", - "towncrier", - "testtools", - "fixtures", - "beautifulsoup4", - "html5lib", - "junitxml", - "tenacity", - "paramiko", - "pytest-timeout", - # Does our OpenMetrics endpoint adhere to the spec: - "prometheus-client == 0.11.0", - ] + tor_requires + i2p_requires, - "tor": tor_requires, - "i2p": i2p_requires, - }, - package_data={"allmydata.web": ["*.xhtml", - "static/*.js", "static/*.png", "static/*.css", - "static/img/*.png", - "static/css/*.css", - ], - "allmydata": ["ported-modules.txt"], - }, - include_package_data=True, - setup_requires=setup_requires, - entry_points = { 'console_scripts': [ 'tahoe = allmydata.scripts.runner:run' ] }, - **setup_args - ) +from setuptools import setup +setup() diff --git a/src/allmydata/__init__.py b/src/allmydata/__init__.py index 333394fc5..42611810b 100644 --- a/src/allmydata/__init__.py +++ b/src/allmydata/__init__.py @@ -16,36 +16,28 @@ if PY2: __all__ = [ "__version__", - "full_version", - "branch", "__appname__", "__full_version__", ] -__version__ = "unknown" -try: - # type ignored as it fails in CI - # (https://app.circleci.com/pipelines/github/tahoe-lafs/tahoe-lafs/1647/workflows/60ae95d4-abe8-492c-8a03-1ad3b9e42ed3/jobs/40972) - from allmydata._version import __version__ # type: ignore -except ImportError: - # We're running in a tree that hasn't run update_version, and didn't - # come with a _version.py, so we don't know what our version is. - # This should not happen very often. - pass - -full_version = "unknown" -branch = "unknown" -try: - # type ignored as it fails in CI - # (https://app.circleci.com/pipelines/github/tahoe-lafs/tahoe-lafs/1647/workflows/60ae95d4-abe8-492c-8a03-1ad3b9e42ed3/jobs/40972) - from allmydata._version import full_version, branch # type: ignore -except ImportError: - # We're running in a tree that hasn't run update_version, and didn't - # come with a _version.py, so we don't know what our full version or - # branch is. This should not happen very often. - pass +def _discover_version(): + try: + from allmydata._version import version + except ImportError: + # Perhaps we're running from a git checkout where the _version.py file + # hasn't been generated yet. Try to discover the version using git + # information instead. + try: + import setuptools_scm + return setuptools_scm.get_version() + except Exception: + return "unknown" + else: + return version __appname__ = "tahoe-lafs" +__version__ = _discover_version() +del _discover_version # __full_version__ is the one that you ought to use when identifying yourself # in the "application" part of the Tahoe versioning scheme: diff --git a/tox.ini b/tox.ini index 610570be5..3aa6116b8 100644 --- a/tox.ini +++ b/tox.ini @@ -148,8 +148,8 @@ commands = # If towncrier.check fails, you forgot to add a towncrier news # fragment explaining the change in this branch. Create one at # `newsfragments/.` with some text for the news - # file. See towncrier.toml for legal values. - python -m towncrier.check --config towncrier.toml + # file. See pyproject.toml for legal values. + python -m towncrier.check --config pyproject.toml [testenv:typechecks] @@ -177,7 +177,7 @@ deps = certifi towncrier==21.3.0 commands = - python -m towncrier --draft --config towncrier.toml + python -m towncrier --draft --config pyproject.toml [testenv:news] # On macOS, git invoked from Tox needs $HOME. @@ -189,7 +189,7 @@ deps = certifi towncrier==21.3.0 commands = - python -m towncrier --yes --config towncrier.toml + python -m towncrier --yes --config pyproject.toml # commit the changes git commit -m "update NEWS.txt for release" From efc9dc831bb1f0f2915657294084bc4fb7e84d91 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 14 Oct 2021 11:01:37 -0400 Subject: [PATCH 264/269] Revert "a stab at using setup.cfg and setuptools_scm" This reverts commit 68e8e0a7d5b88568bd01c1a14957f972830d1f54. --- CLASSIFIERS.txt | 29 --- setup.cfg | 215 +--------------- setup.py | 429 ++++++++++++++++++++++++++++++- src/allmydata/__init__.py | 40 +-- pyproject.toml => towncrier.toml | 10 - tox.ini | 8 +- 6 files changed, 462 insertions(+), 269 deletions(-) delete mode 100644 CLASSIFIERS.txt rename pyproject.toml => towncrier.toml (80%) diff --git a/CLASSIFIERS.txt b/CLASSIFIERS.txt deleted file mode 100644 index 7aa2e35b1..000000000 --- a/CLASSIFIERS.txt +++ /dev/null @@ -1,29 +0,0 @@ -Development Status :: 5 - Production/Stable -Environment :: Console -Environment :: Web Environment -License :: OSI Approved :: GNU General Public License (GPL) -License :: DFSG approved -License :: Other/Proprietary License -Intended Audience :: Developers -Intended Audience :: End Users/Desktop -Intended Audience :: System Administrators -Operating System :: Microsoft -Operating System :: Microsoft :: Windows -Operating System :: Unix -Operating System :: POSIX :: Linux -Operating System :: POSIX -Operating System :: MacOS :: MacOS X -Operating System :: OS Independent -Natural Language :: English -Programming Language :: C -Programming Language :: Python -Programming Language :: Python :: 2 -Programming Language :: Python :: 2.7 -Topic :: Utilities -Topic :: System :: Systems Administration -Topic :: System :: Filesystems -Topic :: System :: Distributed Computing -Topic :: Software Development :: Libraries -Topic :: System :: Archiving :: Backup -Topic :: System :: Archiving :: Mirroring -Topic :: System :: Archiving diff --git a/setup.cfg b/setup.cfg index 21e1fb663..f4539279e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,211 +1,10 @@ -# https://setuptools.pypa.io/en/latest/userguide/declarative_config.html#declarative-config -[metadata] -name = tahoe-lafs - -[options] -version = attr:allmydata._version.version -description = secure, decentralized, fault-tolerant file store -long_description = file: README.rst -author = the Tahoe-LAFS project -author_email = tahoe-dev@lists.tahoe-lafs.org -project_urls = - website=https://tahoe-lafs.org/ - documentation=https://tahoe-lafs.readthedocs.org/ - source=https://github.com/tahoe-lafs/tahoe-lafs -classifiers = file: CLASSIFIERS.txt -license_files = COPYING.GPL, COPYING.TGPPL.rst -# tell setuptools to find our package source files automatically -packages = find: -# find packages beneath the src directory -package_dir= - =src -include_package_data = True - -# We support Python 2.7 and many newer versions of Python 3. -python_requires = >=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.* - -install_requires = - # we don't need much out of setuptools but the version checking stuff - # needs pkg_resources and PEP 440 version specifiers. - setuptools >= 28.8.0 - - zfec >= 1.1.0 - - # zope.interface >= 3.6.0 is required for Twisted >= 12.1.0. - zope.interface >= 3.6.0 - - # * foolscap < 0.5.1 had a performance bug which spent O(N**2) CPU for - # transferring large mutable files of size N. - # * foolscap < 0.6 is incompatible with Twisted 10.2.0. - # * foolscap 0.6.1 quiets a DeprecationWarning. - # * foolscap < 0.6.3 is incompatible with Twisted 11.1.0 and newer. - # * foolscap 0.8.0 generates 2048-bit RSA-with-SHA-256 signatures, - # rather than 1024-bit RSA-with-MD5. This also allows us to work - # with a FIPS build of OpenSSL. - # * foolscap >= 0.12.3 provides tcp/tor/i2p connection handlers we need, - # and allocate_tcp_port - # * foolscap >= 0.12.5 has ConnectionInfo and ReconnectionInfo - # * foolscap >= 0.12.6 has an i2p.sam_endpoint() that takes kwargs - # * foolscap 0.13.2 drops i2p support completely - # * foolscap >= 21.7 is necessary for Python 3 with i2p support. - foolscap == 0.13.1 ; python_version < '3.0' - foolscap >= 21.7.0 ; python_version > '3.0' - - # * cryptography 2.6 introduced some ed25519 APIs we rely on. Note that - # Twisted[conch] also depends on cryptography and Twisted[tls] - # transitively depends on cryptography. So it's anyone's guess what - # version of cryptography will *really* be installed. - cryptography >= 2.6 - - # * The SFTP frontend depends on Twisted 11.0.0 to fix the SSH server - # rekeying bug - # * The SFTP frontend and manhole depend on the conch extra. However, we - # can't explicitly declare that without an undesirable dependency on gmpy, - # as explained in ticket #2740. - # * Due to a setuptools bug, we need to declare a dependency on the tls - # extra even though we only depend on it via foolscap. - # * Twisted >= 15.1.0 is the first version that provided the [tls] extra. - # * Twisted-16.1.0 fixes https://twistedmatrix.com/trac/ticket/8223, - # which otherwise causes test_system to fail (DirtyReactorError, due to - # leftover timers) - # * Twisted-16.4.0 introduces `python -m twisted.trial` which is needed - # for coverage testing - # * Twisted 16.6.0 drops the undesirable gmpy dependency from the conch - # extra, letting us use that extra instead of trying to duplicate its - # dependencies here. Twisted[conch] >18.7 introduces a dependency on - # bcrypt. It is nice to avoid that if the user ends up with an older - # version of Twisted. That's hard to express except by using the extra. - # - # * Twisted 18.4.0 adds `client` and `host` attributes to `Request` in the - # * initializer, needed by logic in our custom `Request` subclass. - # - # In a perfect world, Twisted[conch] would be a dependency of an sftp - # extra. However, pip fails to resolve the dependencies all - # dependencies when asked for Twisted[tls] *and* Twisted[conch]. - # Specifically, Twisted[conch] (as the later requirement) is ignored. - # If there were an Tahoe-LAFS sftp extra that dependended on - # Twisted[conch] and install_requires only included Twisted[tls] then - # `pip install tahoe-lafs[sftp]` would not install requirements - # specified by Twisted[conch]. Since this would be the *whole point* of - # an sftp extra in Tahoe-LAFS, there is no point in having one. - # * Twisted 19.10 introduces Site.getContentFile which we use to get - # temporary upload files placed into a per-node temporary directory. - Twisted[tls,conch] >= 19.10.0 - - PyYAML >= 3.11 - - six >= 1.10.0 - - # for 'tahoe invite' and 'tahoe join' - magic-wormhole >= 0.10.2 - - # Eliot is contemplating dropping Python 2 support. Stick to a version we - # know works on Python 2.7. - eliot ~= 1.7 ; python_version < '3.0' - # On Python 3, we want a new enough version to support custom JSON encoders. - eliot >= 1.13.0 ; python_version > '3.0' - - # Pyrsistent 0.17.0 (which we use by way of Eliot) has dropped - # Python 2 entirely; stick to the version known to work for us. - pyrsistent < 0.17.0 ; python_version < '3.0' - pyrsistent ; python_version > '3.0' - - # A great way to define types of values. - attrs >= 18.2.0 - - # WebSocket library for twisted and asyncio - autobahn >= 19.5.2 - - # Support for Python 3 transition - future >= 0.18.2 - - # Discover local network configuration - netifaces - - # Utility code: - pyutil >= 3.3.0 - - # Linux distribution detection: - distro >= 1.4.0 - - # Backported configparser for Python 2: - configparser ; python_version < '3.0' - - # For the RangeMap datastructure. - collections-extended - - # Duplicate the Twisted pywin32 dependency here. See - # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/2392 for some - # discussion. - pywin32 != 226 ; sys_platform=="win32" - -[options.packages.find] -# inform the setuptools source discovery logic to start in this directory -where = src - -[options.package_data] -allmydata.web = - *.xhtml - static/*.js - static/*.png - static/*.css - static/img/*.png - static/css/*.css - -[options.extras_require] -test = - flake8 - # Pin a specific pyflakes so we don't have different folks - # disagreeing on what is or is not a lint issue. We can bump - # this version from time to time, but we will do it - # intentionally. - pyflakes == 2.2.0 - coverage ~= 5.0 - mock - tox - pytest - pytest-twisted - # XXX: decorator isn't a direct dependency, but pytest-twisted - # depends on decorator, and decorator 5.x isn't compatible with - # Python 2.7. - decorator < 5 - hypothesis >= 3.6.1 - treq - towncrier - testtools - fixtures - beautifulsoup4 - html5lib - junitxml - tenacity - paramiko - pytest-timeout - # Does our OpenMetrics endpoint adhere to the spec: - prometheus-client == 0.11.0 - - # Make sure the tor and i2p tests can run by duplicating the requirements - # for those extras here. - %(tor)s - %(i2p)s - -tor = - # This is exactly what `foolscap[tor]` means but pip resolves the pair of - # dependencies "foolscap[i2p] foolscap[tor]" to "foolscap[i2p]" so we lose - # this if we don't declare it ourselves! - txtorcon >= 0.17.0 - -i2p = - # txi2p has Python 3 support in master branch, but it has not been - # released -- see https://github.com/str4d/txi2p/issues/10. We - # could use a fork for Python 3 until txi2p's maintainers are back - # in action. For Python 2, we could continue using the txi2p - # version about which no one has complained to us so far. - txi2p; python_version < '3.0' - txi2p-tahoe >= 0.3.5; python_version > '3.0' - -[options.entry_points] -console_scripts = - tahoe = allmydata.scripts.runner:run +[aliases] +build = update_version build +sdist = update_version sdist +install = update_version install +develop = update_version develop +bdist_egg = update_version bdist_egg +bdist_wheel = update_version bdist_wheel [flake8] # Enforce all pyflakes constraints, and also prohibit tabs for indentation. diff --git a/setup.py b/setup.py index 8bf1ba938..8c6396937 100644 --- a/setup.py +++ b/setup.py @@ -1,2 +1,427 @@ -from setuptools import setup -setup() +#! /usr/bin/env python +# -*- coding: utf-8 -*- +import sys + +# Tahoe-LAFS -- secure, distributed storage grid +# +# Copyright © 2006-2012 The Tahoe-LAFS Software Foundation +# +# This file is part of Tahoe-LAFS. +# +# See the docs/about.rst file for licensing information. + +import os, subprocess, re +from io import open + +basedir = os.path.dirname(os.path.abspath(__file__)) + +# locate our version number + +def read_version_py(infname): + try: + verstrline = open(infname, "rt").read() + except EnvironmentError: + return None + else: + VSRE = r"^verstr = ['\"]([^'\"]*)['\"]" + mo = re.search(VSRE, verstrline, re.M) + if mo: + return mo.group(1) + +VERSION_PY_FILENAME = 'src/allmydata/_version.py' +version = read_version_py(VERSION_PY_FILENAME) + +install_requires = [ + # we don't need much out of setuptools but the version checking stuff + # needs pkg_resources and PEP 440 version specifiers. + "setuptools >= 28.8.0", + + "zfec >= 1.1.0", + + # zope.interface >= 3.6.0 is required for Twisted >= 12.1.0. + "zope.interface >= 3.6.0", + + # * foolscap < 0.5.1 had a performance bug which spent O(N**2) CPU for + # transferring large mutable files of size N. + # * foolscap < 0.6 is incompatible with Twisted 10.2.0. + # * foolscap 0.6.1 quiets a DeprecationWarning. + # * foolscap < 0.6.3 is incompatible with Twisted 11.1.0 and newer. + # * foolscap 0.8.0 generates 2048-bit RSA-with-SHA-256 signatures, + # rather than 1024-bit RSA-with-MD5. This also allows us to work + # with a FIPS build of OpenSSL. + # * foolscap >= 0.12.3 provides tcp/tor/i2p connection handlers we need, + # and allocate_tcp_port + # * foolscap >= 0.12.5 has ConnectionInfo and ReconnectionInfo + # * foolscap >= 0.12.6 has an i2p.sam_endpoint() that takes kwargs + # * foolscap 0.13.2 drops i2p support completely + # * foolscap >= 21.7 is necessary for Python 3 with i2p support. + "foolscap == 0.13.1 ; python_version < '3.0'", + "foolscap >= 21.7.0 ; python_version > '3.0'", + + # * cryptography 2.6 introduced some ed25519 APIs we rely on. Note that + # Twisted[conch] also depends on cryptography and Twisted[tls] + # transitively depends on cryptography. So it's anyone's guess what + # version of cryptography will *really* be installed. + "cryptography >= 2.6", + + # * The SFTP frontend depends on Twisted 11.0.0 to fix the SSH server + # rekeying bug + # * The SFTP frontend and manhole depend on the conch extra. However, we + # can't explicitly declare that without an undesirable dependency on gmpy, + # as explained in ticket #2740. + # * Due to a setuptools bug, we need to declare a dependency on the tls + # extra even though we only depend on it via foolscap. + # * Twisted >= 15.1.0 is the first version that provided the [tls] extra. + # * Twisted-16.1.0 fixes https://twistedmatrix.com/trac/ticket/8223, + # which otherwise causes test_system to fail (DirtyReactorError, due to + # leftover timers) + # * Twisted-16.4.0 introduces `python -m twisted.trial` which is needed + # for coverage testing + # * Twisted 16.6.0 drops the undesirable gmpy dependency from the conch + # extra, letting us use that extra instead of trying to duplicate its + # dependencies here. Twisted[conch] >18.7 introduces a dependency on + # bcrypt. It is nice to avoid that if the user ends up with an older + # version of Twisted. That's hard to express except by using the extra. + # + # * Twisted 18.4.0 adds `client` and `host` attributes to `Request` in the + # * initializer, needed by logic in our custom `Request` subclass. + # + # In a perfect world, Twisted[conch] would be a dependency of an "sftp" + # extra. However, pip fails to resolve the dependencies all + # dependencies when asked for Twisted[tls] *and* Twisted[conch]. + # Specifically, "Twisted[conch]" (as the later requirement) is ignored. + # If there were an Tahoe-LAFS sftp extra that dependended on + # Twisted[conch] and install_requires only included Twisted[tls] then + # `pip install tahoe-lafs[sftp]` would not install requirements + # specified by Twisted[conch]. Since this would be the *whole point* of + # an sftp extra in Tahoe-LAFS, there is no point in having one. + # * Twisted 19.10 introduces Site.getContentFile which we use to get + # temporary upload files placed into a per-node temporary directory. + "Twisted[tls,conch] >= 19.10.0", + + "PyYAML >= 3.11", + + "six >= 1.10.0", + + # for 'tahoe invite' and 'tahoe join' + "magic-wormhole >= 0.10.2", + + # Eliot is contemplating dropping Python 2 support. Stick to a version we + # know works on Python 2.7. + "eliot ~= 1.7 ; python_version < '3.0'", + # On Python 3, we want a new enough version to support custom JSON encoders. + "eliot >= 1.13.0 ; python_version > '3.0'", + + # Pyrsistent 0.17.0 (which we use by way of Eliot) has dropped + # Python 2 entirely; stick to the version known to work for us. + "pyrsistent < 0.17.0 ; python_version < '3.0'", + "pyrsistent ; python_version > '3.0'", + + # A great way to define types of values. + "attrs >= 18.2.0", + + # WebSocket library for twisted and asyncio + "autobahn >= 19.5.2", + + # Support for Python 3 transition + "future >= 0.18.2", + + # Discover local network configuration + "netifaces", + + # Utility code: + "pyutil >= 3.3.0", + + # Linux distribution detection: + "distro >= 1.4.0", + + # Backported configparser for Python 2: + "configparser ; python_version < '3.0'", + + # For the RangeMap datastructure. + "collections-extended", +] + +setup_requires = [ + 'setuptools >= 28.8.0', # for PEP-440 style versions +] + +tor_requires = [ + # This is exactly what `foolscap[tor]` means but pip resolves the pair of + # dependencies "foolscap[i2p] foolscap[tor]" to "foolscap[i2p]" so we lose + # this if we don't declare it ourselves! + "txtorcon >= 0.17.0", +] + +i2p_requires = [ + # txi2p has Python 3 support in master branch, but it has not been + # released -- see https://github.com/str4d/txi2p/issues/10. We + # could use a fork for Python 3 until txi2p's maintainers are back + # in action. For Python 2, we could continue using the txi2p + # version about which no one has complained to us so far. + "txi2p; python_version < '3.0'", + "txi2p-tahoe >= 0.3.5; python_version > '3.0'", +] + +if len(sys.argv) > 1 and sys.argv[1] == '--fakedependency': + del sys.argv[1] + install_requires += ["fakedependency >= 1.0.0"] + +from setuptools import find_packages, setup +from setuptools import Command +from setuptools.command import install + + +trove_classifiers=[ + "Development Status :: 5 - Production/Stable", + "Environment :: Console", + "Environment :: Web Environment", + "License :: OSI Approved :: GNU General Public License (GPL)", + "License :: DFSG approved", + "License :: Other/Proprietary License", + "Intended Audience :: Developers", + "Intended Audience :: End Users/Desktop", + "Intended Audience :: System Administrators", + "Operating System :: Microsoft", + "Operating System :: Microsoft :: Windows", + "Operating System :: Unix", + "Operating System :: POSIX :: Linux", + "Operating System :: POSIX", + "Operating System :: MacOS :: MacOS X", + "Operating System :: OS Independent", + "Natural Language :: English", + "Programming Language :: C", + "Programming Language :: Python", + "Programming Language :: Python :: 2", + "Programming Language :: Python :: 2.7", + "Topic :: Utilities", + "Topic :: System :: Systems Administration", + "Topic :: System :: Filesystems", + "Topic :: System :: Distributed Computing", + "Topic :: Software Development :: Libraries", + "Topic :: System :: Archiving :: Backup", + "Topic :: System :: Archiving :: Mirroring", + "Topic :: System :: Archiving", + ] + + +GIT_VERSION_BODY = ''' +# This _version.py is generated from git metadata by the tahoe setup.py. + +__pkgname__ = "%(pkgname)s" +real_version = "%(version)s" +full_version = "%(full)s" +branch = "%(branch)s" +verstr = "%(normalized)s" +__version__ = verstr +''' + +def run_command(args, cwd=None): + use_shell = sys.platform == "win32" + try: + p = subprocess.Popen(args, stdout=subprocess.PIPE, cwd=cwd, shell=use_shell) + except EnvironmentError as e: # if this gives a SyntaxError, note that Tahoe-LAFS requires Python 2.7+ + print("Warning: unable to run %r." % (" ".join(args),)) + print(e) + return None + stdout = p.communicate()[0].strip() + if p.returncode != 0: + print("Warning: %r returned error code %r." % (" ".join(args), p.returncode)) + return None + return stdout + + +def versions_from_git(tag_prefix): + # This runs 'git' from the directory that contains this file. That either + # means someone ran a setup.py command (and this code is in + # versioneer.py, thus the containing directory is the root of the source + # tree), or someone ran a project-specific entry point (and this code is + # in _version.py, thus the containing directory is somewhere deeper in + # the source tree). This only gets called if the git-archive 'subst' + # variables were *not* expanded, and _version.py hasn't already been + # rewritten with a short version string, meaning we're inside a checked + # out source tree. + + # versions_from_git (as copied from python-versioneer) returns strings + # like "1.9.0-25-gb73aba9-dirty", which means we're in a tree with + # uncommited changes (-dirty), the latest checkin is revision b73aba9, + # the most recent tag was 1.9.0, and b73aba9 has 25 commits that weren't + # in 1.9.0 . The narrow-minded NormalizedVersion parser that takes our + # output (meant to enable sorting of version strings) refuses most of + # that. Tahoe uses a function named suggest_normalized_version() that can + # handle "1.9.0.post25", so dumb down our output to match. + + try: + source_dir = os.path.dirname(os.path.abspath(__file__)) + except NameError as e: + # some py2exe/bbfreeze/non-CPython implementations don't do __file__ + print("Warning: unable to find version because we could not obtain the source directory.") + print(e) + return {} + stdout = run_command(["git", "describe", "--tags", "--dirty", "--always"], + cwd=source_dir) + if stdout is None: + # run_command already complained. + return {} + stdout = stdout.decode("ascii") + if not stdout.startswith(tag_prefix): + print("Warning: tag %r doesn't start with prefix %r." % (stdout, tag_prefix)) + return {} + version = stdout[len(tag_prefix):] + pieces = version.split("-") + if len(pieces) == 1: + normalized_version = pieces[0] + else: + normalized_version = "%s.post%s" % (pieces[0], pieces[1]) + + stdout = run_command(["git", "rev-parse", "HEAD"], cwd=source_dir) + if stdout is None: + # run_command already complained. + return {} + full = stdout.decode("ascii").strip() + if version.endswith("-dirty"): + full += "-dirty" + normalized_version += ".dev0" + + # Thanks to Jistanidiot at . + stdout = run_command(["git", "rev-parse", "--abbrev-ref", "HEAD"], cwd=source_dir) + branch = (stdout or b"unknown").decode("ascii").strip() + + # this returns native strings (bytes on py2, unicode on py3) + return {"version": version, "normalized": normalized_version, + "full": full, "branch": branch} + +# setup.cfg has an [aliases] section which runs "update_version" before many +# commands (like "build" and "sdist") that need to know our package version +# ahead of time. If you add different commands (or if we forgot some), you +# may need to add it to setup.cfg and configure it to run update_version +# before your command. + +class UpdateVersion(Command): + description = "update _version.py from revision-control metadata" + user_options = install.install.user_options + + def initialize_options(self): + pass + def finalize_options(self): + pass + def run(self): + global version + verstr = version + if os.path.isdir(os.path.join(basedir, ".git")): + verstr = self.try_from_git() + + if verstr: + self.distribution.metadata.version = verstr + else: + print("""\ +******************************************************************** +Warning: no version information found. This may cause tests to fail. +******************************************************************** +""") + + def try_from_git(self): + # If we change the release tag names, we must change this too + versions = versions_from_git("tahoe-lafs-") + + # setup.py might be run by either py2 or py3 (when run by tox, which + # uses py3 on modern debian/ubuntu distros). We want this generated + # file to contain native strings on both (str=bytes in py2, + # str=unicode in py3) + if versions: + body = GIT_VERSION_BODY % { + "pkgname": self.distribution.get_name(), + "version": versions["version"], + "normalized": versions["normalized"], + "full": versions["full"], + "branch": versions["branch"], + } + f = open(VERSION_PY_FILENAME, "wb") + f.write(body.encode("ascii")) + f.close() + print("Wrote normalized version %r into '%s'" % (versions["normalized"], VERSION_PY_FILENAME)) + + return versions.get("normalized", None) + +class PleaseUseTox(Command): + user_options = [] + def initialize_options(self): + pass + def finalize_options(self): + pass + + def run(self): + print("ERROR: Please use 'tox' to run the test suite.") + sys.exit(1) + +setup_args = {} +if version: + setup_args["version"] = version + +setup(name="tahoe-lafs", # also set in __init__.py + description='secure, decentralized, fault-tolerant file store', + long_description=open('README.rst', 'r', encoding='utf-8').read(), + author='the Tahoe-LAFS project', + author_email='tahoe-dev@lists.tahoe-lafs.org', + url='https://tahoe-lafs.org/', + license='GNU GPL', # see README.rst -- there is an alternative licence + cmdclass={"update_version": UpdateVersion, + "test": PleaseUseTox, + }, + package_dir = {'':'src'}, + packages=find_packages('src') + ['allmydata.test.plugins'], + classifiers=trove_classifiers, + # We support Python 2.7, and we're working on support for 3.6 (the + # highest version that PyPy currently supports). + python_requires=">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*", + install_requires=install_requires, + extras_require={ + # Duplicate the Twisted pywin32 dependency here. See + # https://tahoe-lafs.org/trac/tahoe-lafs/ticket/2392 for some + # discussion. + ':sys_platform=="win32"': ["pywin32 != 226"], + "test": [ + "flake8", + # Pin a specific pyflakes so we don't have different folks + # disagreeing on what is or is not a lint issue. We can bump + # this version from time to time, but we will do it + # intentionally. + "pyflakes == 2.2.0", + "coverage ~= 5.0", + "mock", + "tox", + "pytest", + "pytest-twisted", + # XXX: decorator isn't a direct dependency, but pytest-twisted + # depends on decorator, and decorator 5.x isn't compatible with + # Python 2.7. + "decorator < 5", + "hypothesis >= 3.6.1", + "treq", + "towncrier", + "testtools", + "fixtures", + "beautifulsoup4", + "html5lib", + "junitxml", + "tenacity", + "paramiko", + "pytest-timeout", + # Does our OpenMetrics endpoint adhere to the spec: + "prometheus-client == 0.11.0", + ] + tor_requires + i2p_requires, + "tor": tor_requires, + "i2p": i2p_requires, + }, + package_data={"allmydata.web": ["*.xhtml", + "static/*.js", "static/*.png", "static/*.css", + "static/img/*.png", + "static/css/*.css", + ], + "allmydata": ["ported-modules.txt"], + }, + include_package_data=True, + setup_requires=setup_requires, + entry_points = { 'console_scripts': [ 'tahoe = allmydata.scripts.runner:run' ] }, + **setup_args + ) diff --git a/src/allmydata/__init__.py b/src/allmydata/__init__.py index 42611810b..333394fc5 100644 --- a/src/allmydata/__init__.py +++ b/src/allmydata/__init__.py @@ -16,28 +16,36 @@ if PY2: __all__ = [ "__version__", + "full_version", + "branch", "__appname__", "__full_version__", ] -def _discover_version(): - try: - from allmydata._version import version - except ImportError: - # Perhaps we're running from a git checkout where the _version.py file - # hasn't been generated yet. Try to discover the version using git - # information instead. - try: - import setuptools_scm - return setuptools_scm.get_version() - except Exception: - return "unknown" - else: - return version +__version__ = "unknown" +try: + # type ignored as it fails in CI + # (https://app.circleci.com/pipelines/github/tahoe-lafs/tahoe-lafs/1647/workflows/60ae95d4-abe8-492c-8a03-1ad3b9e42ed3/jobs/40972) + from allmydata._version import __version__ # type: ignore +except ImportError: + # We're running in a tree that hasn't run update_version, and didn't + # come with a _version.py, so we don't know what our version is. + # This should not happen very often. + pass + +full_version = "unknown" +branch = "unknown" +try: + # type ignored as it fails in CI + # (https://app.circleci.com/pipelines/github/tahoe-lafs/tahoe-lafs/1647/workflows/60ae95d4-abe8-492c-8a03-1ad3b9e42ed3/jobs/40972) + from allmydata._version import full_version, branch # type: ignore +except ImportError: + # We're running in a tree that hasn't run update_version, and didn't + # come with a _version.py, so we don't know what our full version or + # branch is. This should not happen very often. + pass __appname__ = "tahoe-lafs" -__version__ = _discover_version() -del _discover_version # __full_version__ is the one that you ought to use when identifying yourself # in the "application" part of the Tahoe versioning scheme: diff --git a/pyproject.toml b/towncrier.toml similarity index 80% rename from pyproject.toml rename to towncrier.toml index 7c97001aa..b8b561a98 100644 --- a/pyproject.toml +++ b/towncrier.toml @@ -1,13 +1,3 @@ -# https://setuptools.pypa.io/en/latest/build_meta.html -# https://github.com/pypa/setuptools_scm -[build-system] -requires = ["setuptools>=45", "wheel", "setuptools_scm>=6.2"] -build-backend = "setuptools.build_meta" - -[tool.setuptools_scm] -write_to = "src/allmydata/_version.py" -tag_regex = "^tahoe-lafs-(?P[vV]?\\d+(?:\\.\\d+){0,2}[^\\+]*)(?:\\+.*)?$" - [tool.towncrier] package_dir = "src" package = "allmydata" diff --git a/tox.ini b/tox.ini index 3aa6116b8..610570be5 100644 --- a/tox.ini +++ b/tox.ini @@ -148,8 +148,8 @@ commands = # If towncrier.check fails, you forgot to add a towncrier news # fragment explaining the change in this branch. Create one at # `newsfragments/.` with some text for the news - # file. See pyproject.toml for legal values. - python -m towncrier.check --config pyproject.toml + # file. See towncrier.toml for legal values. + python -m towncrier.check --config towncrier.toml [testenv:typechecks] @@ -177,7 +177,7 @@ deps = certifi towncrier==21.3.0 commands = - python -m towncrier --draft --config pyproject.toml + python -m towncrier --draft --config towncrier.toml [testenv:news] # On macOS, git invoked from Tox needs $HOME. @@ -189,7 +189,7 @@ deps = certifi towncrier==21.3.0 commands = - python -m towncrier --yes --config pyproject.toml + python -m towncrier --yes --config towncrier.toml # commit the changes git commit -m "update NEWS.txt for release" From 22aab98fcf2cf873b87b158e6a819a799f5e912b Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 14 Oct 2021 12:51:24 -0400 Subject: [PATCH 265/269] When callRemoteOnly as removed, these probably should've been changed to return a Deferred. --- src/allmydata/immutable/downloader/share.py | 2 +- src/allmydata/immutable/layout.py | 2 +- src/allmydata/storage_client.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/allmydata/immutable/downloader/share.py b/src/allmydata/immutable/downloader/share.py index 41e11426f..016f1c34d 100644 --- a/src/allmydata/immutable/downloader/share.py +++ b/src/allmydata/immutable/downloader/share.py @@ -475,7 +475,7 @@ class Share(object): # there was corruption somewhere in the given range reason = "corruption in share[%d-%d): %s" % (start, start+offset, str(f.value)) - self._rref.callRemote( + return self._rref.callRemote( "advise_corrupt_share", reason.encode("utf-8") ).addErrback(log.err, "Error from remote call to advise_corrupt_share") diff --git a/src/allmydata/immutable/layout.py b/src/allmydata/immutable/layout.py index 6c7362b8a..79c886237 100644 --- a/src/allmydata/immutable/layout.py +++ b/src/allmydata/immutable/layout.py @@ -254,7 +254,7 @@ class WriteBucketProxy(object): return d def abort(self): - self._rref.callRemote("abort").addErrback(log.err, "Error from remote call to abort an immutable write bucket") + return self._rref.callRemote("abort").addErrback(log.err, "Error from remote call to abort an immutable write bucket") def get_servername(self): return self._server.get_name() diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index ac6c107d5..526e4e70d 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -1017,7 +1017,7 @@ class _StorageServer(object): shnum, reason, ): - self._rref.callRemote( + return self._rref.callRemote( "advise_corrupt_share", share_type, storage_index, From e099bc6736383bb8a66538e35bb1a2aa3c1c6cfc Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 14 Oct 2021 12:52:56 -0400 Subject: [PATCH 266/269] Tests for IStorageServer.advise_corrupt_share. --- src/allmydata/test/test_istorageserver.py | 54 ++++++++++++++++++++--- 1 file changed, 47 insertions(+), 7 deletions(-) diff --git a/src/allmydata/test/test_istorageserver.py b/src/allmydata/test/test_istorageserver.py index 40dcdc8bb..bd056ae13 100644 --- a/src/allmydata/test/test_istorageserver.py +++ b/src/allmydata/test/test_istorageserver.py @@ -20,7 +20,7 @@ if PY2: from random import Random -from twisted.internet.defer import inlineCallbacks +from twisted.internet.defer import inlineCallbacks, returnValue from foolscap.api import Referenceable, RemoteException @@ -405,12 +405,8 @@ class IStorageServerImmutableAPIsTestsMixin(object): ) @inlineCallbacks - def test_bucket_advise_corrupt_share(self): - """ - Calling ``advise_corrupt_share()`` on a bucket returned by - ``IStorageServer.get_buckets()`` does not result in error (other - behavior is opaque at this level of abstraction). - """ + def create_share(self): + """Create a share, return the storage index.""" storage_index = new_storage_index() (_, allocated) = yield self.storage_server.allocate_buckets( storage_index, @@ -423,10 +419,31 @@ class IStorageServerImmutableAPIsTestsMixin(object): yield allocated[0].callRemote("write", 0, b"0123456789") yield allocated[0].callRemote("close") + returnValue(storage_index) + @inlineCallbacks + def test_bucket_advise_corrupt_share(self): + """ + Calling ``advise_corrupt_share()`` on a bucket returned by + ``IStorageServer.get_buckets()`` does not result in error (other + behavior is opaque at this level of abstraction). + """ + storage_index = yield self.create_share() buckets = yield self.storage_server.get_buckets(storage_index) yield buckets[0].callRemote("advise_corrupt_share", b"OH NO") + @inlineCallbacks + def test_advise_corrupt_share(self): + """ + Calling ``advise_corrupt_share()`` on an immutable share does not + result in error (other behavior is opaque at this level of + abstraction). + """ + storage_index = yield self.create_share() + yield self.storage_server.advise_corrupt_share( + b"immutable", storage_index, 0, b"ono" + ) + class IStorageServerMutableAPIsTestsMixin(object): """ @@ -780,6 +797,29 @@ class IStorageServerMutableAPIsTestsMixin(object): {0: [b"abcdefg"], 1: [b"0123456"], 2: [b"9876543"]}, ) + @inlineCallbacks + def test_advise_corrupt_share(self): + """ + Calling ``advise_corrupt_share()`` on a mutable share does not + result in error (other behavior is opaque at this level of + abstraction). + """ + secrets = self.new_secrets() + storage_index = new_storage_index() + (written, _) = yield self.staraw( + storage_index, + secrets, + tw_vectors={ + 0: ([], [(0, b"abcdefg")], 7), + }, + r_vector=[], + ) + self.assertEqual(written, True) + + yield self.storage_server.advise_corrupt_share( + b"mutable", storage_index, 0, b"ono" + ) + class _FoolscapMixin(SystemTestMixin): """Run tests on Foolscap version of ``IStorageServer.""" From 67fb8aeb257ae55ede175eba8f704395b30b0273 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 18 Oct 2021 08:08:01 -0400 Subject: [PATCH 267/269] add the security type --- towncrier.toml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/towncrier.toml b/towncrier.toml index b8b561a98..ae9c9d5a5 100644 --- a/towncrier.toml +++ b/towncrier.toml @@ -12,6 +12,11 @@ "~", ] + [[tool.towncrier.type]] + directory = "security" + name = "Security-related Changes" + showcontent = true + [[tool.towncrier.type]] directory = "incompat" name = "Backwards Incompatible Changes" From a7073fe531f56c1cb033fa815cd34ea1aa4962ca Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 18 Oct 2021 08:08:58 -0400 Subject: [PATCH 268/269] news fragment --- newsfragments/3815.documentation | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/3815.documentation diff --git a/newsfragments/3815.documentation b/newsfragments/3815.documentation new file mode 100644 index 000000000..7abc70bd1 --- /dev/null +++ b/newsfragments/3815.documentation @@ -0,0 +1 @@ +The news file for future releases will include a section for changes with a security impact. \ No newline at end of file From 30ae30e3253d19926a75cd1430fae192b53307be Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Mon, 18 Oct 2021 08:11:49 -0400 Subject: [PATCH 269/269] fix the whitespace :/ --- towncrier.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/towncrier.toml b/towncrier.toml index ae9c9d5a5..e093d0cc4 100644 --- a/towncrier.toml +++ b/towncrier.toml @@ -14,8 +14,8 @@ [[tool.towncrier.type]] directory = "security" - name = "Security-related Changes" - showcontent = true + name = "Security-related Changes" + showcontent = true [[tool.towncrier.type]] directory = "incompat"