From 2205e144f97d8fc14923ee29b0a659419e7cf855 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Fri, 13 Nov 2020 21:06:58 -0500
Subject: [PATCH 01/67] Stop pointing folks at [client]introducer.furl in the
 docs

---
 docs/configuration.rst            | 45 +++++++++++++++++--------------
 docs/historical/configuration.rst |  2 +-
 docs/running.rst                  |  6 ++---
 3 files changed, 29 insertions(+), 24 deletions(-)

diff --git a/docs/configuration.rst b/docs/configuration.rst
index ab4751a04..8583888ca 100644
--- a/docs/configuration.rst
+++ b/docs/configuration.rst
@@ -398,13 +398,13 @@ This section controls *when* Tor and I2P are used. The ``[tor]`` and
 ``[i2p]`` sections (described later) control *how* Tor/I2P connections are
 managed.
 
-All Tahoe nodes need to make a connection to the Introducer; the ``[client]
-introducer.furl`` setting (described below) indicates where the Introducer
-lives. Tahoe client nodes must also make connections to storage servers:
-these targets are specified in announcements that come from the Introducer.
-Both are expressed as FURLs (a Foolscap URL), which include a list of
-"connection hints". Each connection hint describes one (of perhaps many)
-network endpoints where the service might live.
+All Tahoe nodes need to make a connection to the Introducer; the
+``private/introducers.yaml`` file (described below) configures where the
+Introducer lives. Tahoe client nodes must also make connections to storage
+servers: these targets are specified in announcements that come from the
+Introducer.  Both are expressed as FURLs (a Foolscap URL), which include a
+list of "connection hints". Each connection hint describes one (of perhaps
+many) network endpoints where the service might live.
 
 Connection hints include a type, and look like:
 
@@ -580,6 +580,8 @@ Client Configuration
 
 ``introducer.furl = (FURL string, mandatory)``
 
+    DEPRECATED.  See :ref:`introducer-definitions`.
+
     This FURL tells the client how to connect to the introducer. Each
     Tahoe-LAFS grid is defined by an introducer. The introducer's FURL is
     created by the introducer node and written into its private base
@@ -965,29 +967,28 @@ This section describes these other files.
   with as many people as possible, put the empty string (so that
   ``private/convergence`` is a zero-length file).
 
-Additional Introducer Definitions
-=================================
+.. _introducer-definitions:
 
-The ``private/introducers.yaml`` file defines additional Introducers. The
-first introducer is defined in ``tahoe.cfg``, in ``[client]
-introducer.furl``. To use two or more Introducers, choose a locally-unique
-"petname" for each one, then define their FURLs in
-``private/introducers.yaml`` like this::
+Introducer Definitions
+======================
+
+The ``private/introducers.yaml`` file defines Introducers.
+Choose a locally-unique "petname" for each one then define their FURLs in ``private/introducers.yaml`` like this::
 
   introducers:
     petname2:
-      furl: FURL2
+      furl: "FURL2"
     petname3:
-      furl: FURL3
+      furl: "FURL3"
 
 Servers will announce themselves to all configured introducers. Clients will
 merge the announcements they receive from all introducers. Nothing will
 re-broadcast an announcement (i.e. telling introducer 2 about something you
 heard from introducer 1).
 
-If you omit the introducer definitions from both ``tahoe.cfg`` and
-``introducers.yaml``, the node will not use an Introducer at all. Such
-"introducerless" clients must be configured with static servers (described
+If you omit the introducer definitions from ``introducers.yaml``,
+the node will not use an Introducer at all.
+Such "introducerless" clients must be configured with static servers (described
 below), or they will not be able to upload and download files.
 
 Static Server Definitions
@@ -1152,7 +1153,6 @@ a legal one.
   timeout.disconnect = 1800
 
   [client]
-  introducer.furl = pb://ok45ssoklj4y7eok5c3xkmj@tcp:tahoe.example:44801/ii3uumo
   helper.furl = pb://ggti5ssoklj4y7eok5c3xkmj@tcp:helper.tahoe.example:7054/kk8lhr
 
   [storage]
@@ -1163,6 +1163,11 @@ a legal one.
   [helper]
   enabled = True
 
+To be introduced to storage servers, here is a sample ``private/introducers.yaml`` which can be used in conjunction::
+
+  introducers:
+    examplegrid:
+      furl: "pb://ok45ssoklj4y7eok5c3xkmj@tcp:tahoe.example:44801/ii3uumo"
 
 Old Configuration Files
 =======================
diff --git a/docs/historical/configuration.rst b/docs/historical/configuration.rst
index 660bc8489..7d9b9fbe4 100644
--- a/docs/historical/configuration.rst
+++ b/docs/historical/configuration.rst
@@ -20,7 +20,7 @@ Config setting                   File                                 Comment
 ``[node]log_gatherer.furl``      ``BASEDIR/log_gatherer.furl``        (one per line)
 ``[node]timeout.keepalive``      ``BASEDIR/keepalive_timeout``
 ``[node]timeout.disconnect``     ``BASEDIR/disconnect_timeout``
-``[client]introducer.furl``      ``BASEDIR/introducer.furl``
+                                 ``BASEDIR/introducer.furl``          ``BASEDIR/private/introducers.yaml``
 ``[client]helper.furl``          ``BASEDIR/helper.furl``
 ``[client]key_generator.furl``   ``BASEDIR/key_generator.furl``
 ``[client]stats_gatherer.furl``  ``BASEDIR/stats_gatherer.furl``
diff --git a/docs/running.rst b/docs/running.rst
index 2b43adf75..ef6ba42ed 100644
--- a/docs/running.rst
+++ b/docs/running.rst
@@ -65,9 +65,9 @@ Running a Client
 To construct a client node, run “``tahoe create-client``”, which will create
 ``~/.tahoe`` to be the node's base directory. Acquire the ``introducer.furl``
 (see below if you are running your own introducer, or use the one from the
-`TestGrid page`_), and paste it after ``introducer.furl =`` in the
-``[client]`` section of ``~/.tahoe/tahoe.cfg``. Then use “``tahoe run
-~/.tahoe``”. After that, the node should be off and running. The first thing
+`TestGrid page`_), and write it to ``~/.tahoe/private/introducers.yaml``
+(see :ref:`introducer-definitions`). Then use “``tahoe run ~/.tahoe``”.
+After that, the node should be off and running. The first thing
 it will do is connect to the introducer and get itself connected to all other
 nodes on the grid.
 

From bcef851ae04aee3ec0cd15100282516b60a92b5d Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Fri, 13 Nov 2020 21:08:19 -0500
Subject: [PATCH 02/67] news fragment

---
 newsfragments/3504.configuration | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 newsfragments/3504.configuration

diff --git a/newsfragments/3504.configuration b/newsfragments/3504.configuration
new file mode 100644
index 000000000..9ff74482c
--- /dev/null
+++ b/newsfragments/3504.configuration
@@ -0,0 +1 @@
+The ``[client]introducer.furl`` configuration item is now deprecated in favor of the ``private/introducers.yaml`` file.
\ No newline at end of file

From bef5ccd0cab356fc534b16b65b8a6af960fcfea2 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Sat, 14 Nov 2020 09:12:14 -0500
Subject: [PATCH 03/67] Move the introducer config reading code into _Config

---
 src/allmydata/client.py | 45 ++--------------------------
 src/allmydata/node.py   | 65 +++++++++++++++++++++++++++++++++++++++++
 2 files changed, 68 insertions(+), 42 deletions(-)

diff --git a/src/allmydata/client.py b/src/allmydata/client.py
index a768ba354..8e9f8f9e8 100644
--- a/src/allmydata/client.py
+++ b/src/allmydata/client.py
@@ -1,7 +1,6 @@
 import os, stat, time, weakref
 from base64 import urlsafe_b64encode
 from functools import partial
-from errno import ENOENT, EPERM
 
 # On Python 2 this will be the backported package:
 from configparser import NoSectionError
@@ -464,51 +463,13 @@ def create_introducer_clients(config, main_tub, _introducer_factory=None):
     # we return this list
     introducer_clients = []
 
-    introducers_yaml_filename = config.get_private_path("introducers.yaml")
-    introducers_filepath = FilePath(introducers_yaml_filename)
+    introducers = config.get_introducer_configuration()
 
-    try:
-        with introducers_filepath.open() as f:
-            introducers_yaml = yamlutil.safe_load(f)
-            if introducers_yaml is None:
-                raise EnvironmentError(
-                    EPERM,
-                    "Can't read '{}'".format(introducers_yaml_filename),
-                    introducers_yaml_filename,
-                )
-            introducers = introducers_yaml.get("introducers", {})
-            log.msg(
-                "found {} introducers in private/introducers.yaml".format(
-                    len(introducers),
-                )
-            )
-    except EnvironmentError as e:
-        if e.errno != ENOENT:
-            raise
-        introducers = {}
-
-    if "default" in introducers.keys():
-        raise ValueError(
-            "'default' introducer furl cannot be specified in introducers.yaml;"
-            " please fix impossible configuration."
-        )
-
-    # read furl from tahoe.cfg
-    tahoe_cfg_introducer_furl = config.get_config("client", "introducer.furl", None)
-    if tahoe_cfg_introducer_furl == "None":
-        raise ValueError(
-            "tahoe.cfg has invalid 'introducer.furl = None':"
-            " to disable it, use 'introducer.furl ='"
-            " or omit the key entirely"
-        )
-    if tahoe_cfg_introducer_furl:
-        introducers[u'default'] = {'furl':tahoe_cfg_introducer_furl}
-
-    for petname, introducer in introducers.items():
+    for petname, introducer_furl in introducers.items():
         introducer_cache_filepath = FilePath(config.get_private_path("introducer_{}_cache.yaml".format(petname)))
         ic = _introducer_factory(
             main_tub,
-            introducer['furl'].encode("ascii"),
+            introducer_furl.encode("ascii"),
             config.nickname,
             str(allmydata.__full_version__),
             str(_Client.OLDEST_SUPPORTED_VERSION),
diff --git a/src/allmydata/node.py b/src/allmydata/node.py
index 9e7143fd4..7dc579a83 100644
--- a/src/allmydata/node.py
+++ b/src/allmydata/node.py
@@ -21,10 +21,13 @@ import types
 import errno
 import tempfile
 from base64 import b32decode, b32encode
+from errno import ENOENT, EPERM
+from warnings import warn
 
 # On Python 2 this will be the backported package.
 import configparser
 
+from twisted.python.filepath import FilePath
 from twisted.python import log as twlog
 from twisted.application import service
 from twisted.python.failure import Failure
@@ -37,6 +40,9 @@ from allmydata.util.assertutil import _assert
 from allmydata.util.fileutil import abspath_expanduser_unicode
 from allmydata.util.encodingutil import get_filesystem_encoding, quote_output
 from allmydata.util import configutil
+from allmydata.util.yamlutil import (
+    safe_load,
+)
 
 def _common_valid_config():
     return configutil.ValidConfiguration({
@@ -428,6 +434,65 @@ class _Config(object):
             os.path.join(self._basedir, *args)
         )
 
+    def get_introducer_configuration(self):
+        """
+        Get configuration for introducers.
+
+        :return {unicode: unicode}: A mapping from introducer petname to the
+            introducer's fURL.
+        """
+        introducers_yaml_filename = self.get_private_path("introducers.yaml")
+        introducers_filepath = FilePath(introducers_yaml_filename)
+
+        try:
+            with introducers_filepath.open() as f:
+                introducers_yaml = safe_load(f)
+                if introducers_yaml is None:
+                    raise EnvironmentError(
+                        EPERM,
+                        "Can't read '{}'".format(introducers_yaml_filename),
+                        introducers_yaml_filename,
+                    )
+                introducers = {
+                    petname: config["furl"]
+                    for petname, config
+                    in introducers_yaml.get("introducers", {}).items()
+                }
+                log.msg(
+                    "found {} introducers in private/introducers.yaml".format(
+                        len(introducers),
+                    )
+                )
+        except EnvironmentError as e:
+            if e.errno != ENOENT:
+                raise
+            introducers = {}
+
+        if "default" in introducers.keys():
+            raise ValueError(
+                "'default' introducer furl cannot be specified in introducers.yaml;"
+                " please fix impossible configuration."
+            )
+
+        # read furl from tahoe.cfg
+        tahoe_cfg_introducer_furl = self.get_config("client", "introducer.furl", None)
+        if tahoe_cfg_introducer_furl == "None":
+            raise ValueError(
+                "tahoe.cfg has invalid 'introducer.furl = None':"
+                " to disable it, use 'introducer.furl ='"
+                " or omit the key entirely"
+            )
+        if tahoe_cfg_introducer_furl:
+            warn(
+                "tahoe.cfg [client]introducer.furl is deprecated; "
+                "use private/introducers.yaml instead.",
+                category=DeprecationWarning,
+                stacklevel=-1,
+            )
+            introducers['default'] = tahoe_cfg_introducer_furl
+
+        return introducers
+
 
 def create_tub_options(config):
     """

From e0f69dcfcf26e5faabcb3fca13d69ba3e4567602 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Sat, 14 Nov 2020 09:26:07 -0500
Subject: [PATCH 04/67] Get the path manipulation into _Config too

---
 src/allmydata/client.py |  7 +++----
 src/allmydata/node.py   | 15 ++++++++++++---
 2 files changed, 15 insertions(+), 7 deletions(-)

diff --git a/src/allmydata/client.py b/src/allmydata/client.py
index 8e9f8f9e8..ed65917ed 100644
--- a/src/allmydata/client.py
+++ b/src/allmydata/client.py
@@ -465,17 +465,16 @@ def create_introducer_clients(config, main_tub, _introducer_factory=None):
 
     introducers = config.get_introducer_configuration()
 
-    for petname, introducer_furl in introducers.items():
-        introducer_cache_filepath = FilePath(config.get_private_path("introducer_{}_cache.yaml".format(petname)))
+    for petname, (furl, cache_path) in introducers.items():
         ic = _introducer_factory(
             main_tub,
-            introducer_furl.encode("ascii"),
+            furl.encode("ascii"),
             config.nickname,
             str(allmydata.__full_version__),
             str(_Client.OLDEST_SUPPORTED_VERSION),
             list(node.get_app_versions()),
             partial(_sequencer, config),
-            introducer_cache_filepath,
+            cache_path,
         )
         introducer_clients.append(ic)
     return introducer_clients
diff --git a/src/allmydata/node.py b/src/allmydata/node.py
index 7dc579a83..7b8c5d65a 100644
--- a/src/allmydata/node.py
+++ b/src/allmydata/node.py
@@ -438,12 +438,17 @@ class _Config(object):
         """
         Get configuration for introducers.
 
-        :return {unicode: unicode}: A mapping from introducer petname to the
-            introducer's fURL.
+        :return {unicode: (unicode, FilePath)}: A mapping from introducer
+            petname to a tuple of the introducer's fURL and local cache path.
         """
         introducers_yaml_filename = self.get_private_path("introducers.yaml")
         introducers_filepath = FilePath(introducers_yaml_filename)
 
+        def get_cache_filepath(petname):
+            return FilePath(
+                self.get_private_path("introducer_{}_cache.yaml".format(petname)),
+            )
+
         try:
             with introducers_filepath.open() as f:
                 introducers_yaml = safe_load(f)
@@ -491,7 +496,11 @@ class _Config(object):
             )
             introducers['default'] = tahoe_cfg_introducer_furl
 
-        return introducers
+        return {
+            petname: (furl, get_cache_filepath(petname))
+            for (petname, furl)
+            in introducers.items()
+        }
 
 
 def create_tub_options(config):

From 25666ee49c2b5a6bb238aa5c221013346fabd662 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Sat, 14 Nov 2020 09:57:17 -0500
Subject: [PATCH 05/67] Get rid of [client]introducer.furl from test_client

---
 src/allmydata/test/test_client.py | 57 +++++++++++++++++++++----------
 1 file changed, 39 insertions(+), 18 deletions(-)

diff --git a/src/allmydata/test/test_client.py b/src/allmydata/test/test_client.py
index 0f0648a4c..8e6bcd6ec 100644
--- a/src/allmydata/test/test_client.py
+++ b/src/allmydata/test/test_client.py
@@ -75,13 +75,7 @@ from .matchers import (
 
 SOME_FURL = b"pb://abcde@nowhere/fake"
 
-BASECONFIG = ("[client]\n"
-              "introducer.furl = \n"
-              )
-
-BASECONFIG_I = ("[client]\n"
-              "introducer.furl = %s\n"
-              )
+BASECONFIG = "[client]\n"
 
 class Basic(testutil.ReallyEqualMixin, unittest.TestCase):
     def test_loadable(self):
@@ -660,6 +654,22 @@ def flush_but_dont_ignore(res):
     return d
 
 
+def write_introducer(basedir, petname, furl):
+    """
+    Overwrite the node's ``introducers.yaml`` with a file containing the given
+    introducer information.
+    """
+    FilePath(basedir).child(b"private").child(b"introducers.yaml").setContent(
+        safe_dump({
+            "introducers": {
+                petname: {
+                    "furl": furl,
+                },
+            },
+        }),
+    )
+
+
 class AnonymousStorage(SyncTestCase):
     """
     Tests for behaviors of the client object with respect to the anonymous
@@ -672,10 +682,11 @@ class AnonymousStorage(SyncTestCase):
         """
         basedir = self.id()
         os.makedirs(basedir + b"/private")
+        write_introducer(basedir, "someintroducer", SOME_FURL)
         config = client.config_from_string(
             basedir,
             "tub.port",
-            BASECONFIG_I % (SOME_FURL,) + (
+            BASECONFIG + (
                 "[storage]\n"
                 "enabled = true\n"
                 "anonymous = true\n"
@@ -703,10 +714,11 @@ class AnonymousStorage(SyncTestCase):
         """
         basedir = self.id()
         os.makedirs(basedir + b"/private")
+        write_introducer(basedir, "someintroducer", SOME_FURL)
         config = client.config_from_string(
             basedir,
             "tub.port",
-            BASECONFIG_I % (SOME_FURL,) + (
+            BASECONFIG + (
                 "[storage]\n"
                 "enabled = true\n"
                 "anonymous = false\n"
@@ -743,7 +755,7 @@ class AnonymousStorage(SyncTestCase):
         enabled_config = client.config_from_string(
             basedir,
             "tub.port",
-            BASECONFIG_I % (SOME_FURL,) + (
+            BASECONFIG + (
                 "[storage]\n"
                 "enabled = true\n"
                 "anonymous = true\n"
@@ -767,7 +779,7 @@ class AnonymousStorage(SyncTestCase):
         disabled_config = client.config_from_string(
             basedir,
             "tub.port",
-            BASECONFIG_I % (SOME_FURL,) + (
+            BASECONFIG + (
                 "[storage]\n"
                 "enabled = true\n"
                 "anonymous = false\n"
@@ -953,19 +965,25 @@ class Run(unittest.TestCase, testutil.StallMixin):
 
     @defer.inlineCallbacks
     def test_loadable(self):
+        """
+        A configuration consisting only of an introducer can be turned into a
+        client node.
+        """
         basedir = "test_client.Run.test_loadable"
-        os.mkdir(basedir)
+        os.makedirs(basedir + b"/private")
         dummy = "pb://wl74cyahejagspqgy4x5ukrvfnevlknt@127.0.0.1:58889/bogus"
-        fileutil.write(os.path.join(basedir, "tahoe.cfg"), BASECONFIG_I % dummy)
+        write_introducer(basedir, "someintroducer", dummy)
+        fileutil.write(os.path.join(basedir, "tahoe.cfg"), BASECONFIG)
         fileutil.write(os.path.join(basedir, client._Client.EXIT_TRIGGER_FILE), "")
         yield client.create_client(basedir)
 
     @defer.inlineCallbacks
     def test_reloadable(self):
         basedir = "test_client.Run.test_reloadable"
-        os.mkdir(basedir)
+        os.makedirs(basedir + b"/private")
         dummy = "pb://wl74cyahejagspqgy4x5ukrvfnevlknt@127.0.0.1:58889/bogus"
-        fileutil.write(os.path.join(basedir, "tahoe.cfg"), BASECONFIG_I % dummy)
+        write_introducer(basedir, "someintroducer", dummy)
+        fileutil.write(os.path.join(basedir, "tahoe.cfg"), BASECONFIG)
         c1 = yield client.create_client(basedir)
         c1.setServiceParent(self.sparent)
 
@@ -1129,10 +1147,16 @@ class StorageAnnouncementTests(SyncTestCase):
         super(StorageAnnouncementTests, self).setUp()
         self.basedir = self.useFixture(TempDir()).path
         create_node_dir(self.basedir, u"")
+        # Write an introducer configuration or we can't observer
+        # announcements.
+        write_introducer(self.basedir, "someintroducer", SOME_FURL)
 
 
     def get_config(self, storage_enabled, more_storage="", more_sections=""):
         return """
+[client]
+# Empty
+
 [node]
 tub.location = tcp:192.0.2.0:1234
 
@@ -1140,9 +1164,6 @@ tub.location = tcp:192.0.2.0:1234
 enabled = {storage_enabled}
 {more_storage}
 
-[client]
-introducer.furl = pb://abcde@nowhere/fake
-
 {more_sections}
 """.format(
     storage_enabled=storage_enabled,

From b202f81fd1b46fef73c744b05a545c42123381f8 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Sat, 14 Nov 2020 10:23:07 -0500
Subject: [PATCH 06/67] move config helper to shared location

---
 src/allmydata/test/common.py      | 20 ++++++++++++++++++++
 src/allmydata/test/test_client.py | 17 +----------------
 2 files changed, 21 insertions(+), 16 deletions(-)

diff --git a/src/allmydata/test/common.py b/src/allmydata/test/common.py
index 1cf1d6428..2bf3ca50d 100644
--- a/src/allmydata/test/common.py
+++ b/src/allmydata/test/common.py
@@ -32,6 +32,10 @@ import attr
 
 import treq
 
+from yaml import (
+    safe_dump,
+)
+
 from zope.interface import implementer
 
 from testtools import (
@@ -100,6 +104,22 @@ EMPTY_CLIENT_CONFIG = config_from_string(
 )
 
 
+def write_introducer(basedir, petname, furl):
+    """
+    Overwrite the node's ``introducers.yaml`` with a file containing the given
+    introducer information.
+    """
+    FilePath(basedir).child(b"private").child(b"introducers.yaml").setContent(
+        safe_dump({
+            "introducers": {
+                petname: {
+                    "furl": furl,
+                },
+            },
+        }),
+    )
+
+
 @attr.s
 class MemoryIntroducerClient(object):
     """
diff --git a/src/allmydata/test/test_client.py b/src/allmydata/test/test_client.py
index 8e6bcd6ec..b76047be0 100644
--- a/src/allmydata/test/test_client.py
+++ b/src/allmydata/test/test_client.py
@@ -66,6 +66,7 @@ from .common import (
     UseTestPlugins,
     MemoryIntroducerClient,
     get_published_announcements,
+    write_introducer,
 )
 from .matchers import (
     MatchesSameElements,
@@ -654,22 +655,6 @@ def flush_but_dont_ignore(res):
     return d
 
 
-def write_introducer(basedir, petname, furl):
-    """
-    Overwrite the node's ``introducers.yaml`` with a file containing the given
-    introducer information.
-    """
-    FilePath(basedir).child(b"private").child(b"introducers.yaml").setContent(
-        safe_dump({
-            "introducers": {
-                petname: {
-                    "furl": furl,
-                },
-            },
-        }),
-    )
-
-
 class AnonymousStorage(SyncTestCase):
     """
     Tests for behaviors of the client object with respect to the anonymous

From 0664416f6594dea2e6f25d7d57805c0cf63eb3a5 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Sat, 14 Nov 2020 10:23:28 -0500
Subject: [PATCH 07/67] Remove [client]introducer.furl from test_introducer

---
 src/allmydata/node.py                 | 11 +++++------
 src/allmydata/test/test_introducer.py | 17 ++++++++++-------
 2 files changed, 15 insertions(+), 13 deletions(-)

diff --git a/src/allmydata/node.py b/src/allmydata/node.py
index 7b8c5d65a..87a9579d8 100644
--- a/src/allmydata/node.py
+++ b/src/allmydata/node.py
@@ -473,12 +473,6 @@ class _Config(object):
                 raise
             introducers = {}
 
-        if "default" in introducers.keys():
-            raise ValueError(
-                "'default' introducer furl cannot be specified in introducers.yaml;"
-                " please fix impossible configuration."
-            )
-
         # read furl from tahoe.cfg
         tahoe_cfg_introducer_furl = self.get_config("client", "introducer.furl", None)
         if tahoe_cfg_introducer_furl == "None":
@@ -494,6 +488,11 @@ class _Config(object):
                 category=DeprecationWarning,
                 stacklevel=-1,
             )
+            if "default" in introducers:
+                raise ValueError(
+                    "'default' introducer furl cannot be specified in tahoe.cfg and introducers.yaml;"
+                    " please fix impossible configuration."
+                )
             introducers['default'] = tahoe_cfg_introducer_furl
 
         return {
diff --git a/src/allmydata/test/test_introducer.py b/src/allmydata/test/test_introducer.py
index d99e18c4a..4ccabd4e4 100644
--- a/src/allmydata/test/test_introducer.py
+++ b/src/allmydata/test/test_introducer.py
@@ -40,7 +40,8 @@ from allmydata.util.iputil import (
     listenOnUnused,
 )
 import allmydata.test.common_util as testutil
-from allmydata.test.common import (
+from .common import (
+    write_introducer,
     SyncTestCase,
     AsyncTestCase,
     AsyncBrokenTestCase,
@@ -788,8 +789,13 @@ class Announcements(AsyncTestCase):
 
     @defer.inlineCallbacks
     def test_client_cache(self):
+        """
+        Announcements received by an introducer client are written to that
+        introducer client's cache file.
+        """
         basedir = "introducer/ClientSeqnums/test_client_cache_1"
-        fileutil.make_dirs(basedir)
+        fileutil.make_dirs(basedir + b"/private")
+        write_introducer(basedir, "default", "nope")
         cache_filepath = FilePath(os.path.join(basedir, "private",
                                                "introducer_default_cache.yaml"))
 
@@ -798,8 +804,6 @@ class Announcements(AsyncTestCase):
         # until the introducer connection is established). To avoid getting
         # confused by this, disable storage.
         with open(os.path.join(basedir, "tahoe.cfg"), "w") as f:
-            f.write("[client]\n")
-            f.write("introducer.furl = nope\n")
             f.write("[storage]\n")
             f.write("enabled = false\n")
 
@@ -880,14 +884,13 @@ class ClientSeqnums(AsyncBrokenTestCase):
     @defer.inlineCallbacks
     def test_client(self):
         basedir = "introducer/ClientSeqnums/test_client"
-        fileutil.make_dirs(basedir)
+        fileutil.make_dirs(basedir + b"/private")
+        write_introducer(basedir, "default", "nope")
         # if storage is enabled, the Client will publish its storage server
         # during startup (although the announcement will wait in a queue
         # until the introducer connection is established). To avoid getting
         # confused by this, disable storage.
         f = open(os.path.join(basedir, "tahoe.cfg"), "w")
-        f.write("[client]\n")
-        f.write("introducer.furl = nope\n")
         f.write("[storage]\n")
         f.write("enabled = false\n")
         f.close()

From fabcc079c51e1048f867301087e8fc495d983b94 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Sat, 14 Nov 2020 10:58:01 -0500
Subject: [PATCH 08/67] we're not testing the yaml library

---
 src/allmydata/test/test_multi_introducers.py | 12 ------------
 1 file changed, 12 deletions(-)

diff --git a/src/allmydata/test/test_multi_introducers.py b/src/allmydata/test/test_multi_introducers.py
index 34e6e5d96..2d057e604 100644
--- a/src/allmydata/test/test_multi_introducers.py
+++ b/src/allmydata/test/test_multi_introducers.py
@@ -52,18 +52,6 @@ class MultiIntroTests(unittest.TestCase):
         # assertions
         self.failUnlessEqual(ic_count, 3)
 
-    @defer.inlineCallbacks
-    def test_introducer_count_commented(self):
-        """ Ensure that the Client creates same number of introducer clients
-        as found in "basedir/private/introducers" config file when there is one
-        commented."""
-        self.yaml_path.setContent(INTRODUCERS_CFG_FURLS_COMMENTED)
-        # get a client and count of introducer_clients
-        myclient = yield create_client(self.basedir)
-        ic_count = len(myclient.introducer_clients)
-
-        # assertions
-        self.failUnlessEqual(ic_count, 2)
 
     @defer.inlineCallbacks
     def test_read_introducer_furl_from_tahoecfg(self):

From b181b577e8f1a123a5847411df0ff2dc69878c80 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Sat, 14 Nov 2020 11:09:40 -0500
Subject: [PATCH 09/67] Remove [client]introducer.furl from
 test_multi_introducers (mostly)

Leave in this one test to demonstrate the deprecated functionality still
works, until we delete it entirely.
---
 src/allmydata/test/test_multi_introducers.py | 50 +++++++++++++-------
 1 file changed, 34 insertions(+), 16 deletions(-)

diff --git a/src/allmydata/test/test_multi_introducers.py b/src/allmydata/test/test_multi_introducers.py
index 2d057e604..36801c740 100644
--- a/src/allmydata/test/test_multi_introducers.py
+++ b/src/allmydata/test/test_multi_introducers.py
@@ -24,9 +24,6 @@ class MultiIntroTests(unittest.TestCase):
         config = {'hide-ip':False, 'listen': 'tcp',
                   'port': None, 'location': None, 'hostname': 'example.net'}
         write_node_config(c, config)
-        fake_furl = "furl1"
-        c.write("[client]\n")
-        c.write("introducer.furl = %s\n" % fake_furl)
         c.write("[storage]\n")
         c.write("enabled = false\n")
         c.close()
@@ -36,8 +33,10 @@ class MultiIntroTests(unittest.TestCase):
 
     @defer.inlineCallbacks
     def test_introducer_count(self):
-        """ Ensure that the Client creates same number of introducer clients
-        as found in "basedir/private/introducers" config file. """
+        """
+        If there are two introducers configured in ``introducers.yaml`` then
+        ``Client`` creates two introducer clients.
+        """
         connections = {
             'introducers': {
                 u'intro1':{ 'furl': 'furl1' },
@@ -50,13 +49,13 @@ class MultiIntroTests(unittest.TestCase):
         ic_count = len(myclient.introducer_clients)
 
         # assertions
-        self.failUnlessEqual(ic_count, 3)
-
+        self.failUnlessEqual(ic_count, len(connections["introducers"]))
 
     @defer.inlineCallbacks
     def test_read_introducer_furl_from_tahoecfg(self):
-        """ Ensure that the Client reads the introducer.furl config item from
-        the tahoe.cfg file. """
+        """
+        The deprecated [client]introducer.furl item is still read and respected.
+        """
         # create a custom tahoe.cfg
         c = open(os.path.join(self.basedir, "tahoe.cfg"), "w")
         config = {'hide-ip':False, 'listen': 'tcp',
@@ -75,20 +74,41 @@ class MultiIntroTests(unittest.TestCase):
 
         # assertions
         self.failUnlessEqual(fake_furl, tahoe_cfg_furl)
+        self.assertEqual(
+            list(
+                warning["message"]
+                for warning
+                in self.flushWarnings()
+                if warning["category"] is DeprecationWarning
+            ),
+            ["tahoe.cfg [client]introducer.furl is deprecated; "
+             "use private/introducers.yaml instead."],
+        )
 
     @defer.inlineCallbacks
     def test_reject_default_in_yaml(self):
-        connections = {'introducers': {
-            u'default': { 'furl': 'furl1' },
-            }}
+        """
+        If an introducer is configured in tahoe.cfg then a "default" introducer in
+        introducers.yaml is rejected.
+        """
+        connections = {
+            'introducers': {
+                u'default': { 'furl': 'furl1' },
+            },
+        }
         self.yaml_path.setContent(yamlutil.safe_dump(connections))
+        FilePath(self.basedir).child("tahoe.cfg").setContent(
+            "[client]\n"
+            "introducer.furl = furl1\n"
+        )
+
         with self.assertRaises(ValueError) as ctx:
             yield create_client(self.basedir)
 
         self.assertEquals(
             str(ctx.exception),
-            "'default' introducer furl cannot be specified in introducers.yaml; please "
-            "fix impossible configuration.",
+            "'default' introducer furl cannot be specified in tahoe.cfg and introducers.yaml; "
+            "please fix impossible configuration.",
         )
 
 SIMPLE_YAML = """
@@ -114,8 +134,6 @@ class NoDefault(unittest.TestCase):
         config = {'hide-ip':False, 'listen': 'tcp',
                   'port': None, 'location': None, 'hostname': 'example.net'}
         write_node_config(c, config)
-        c.write("[client]\n")
-        c.write("# introducer.furl =\n") # omit default
         c.write("[storage]\n")
         c.write("enabled = false\n")
         c.close()

From b6bebc514a8efe0b1ad2237dca91bc10da434bf2 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Sat, 14 Nov 2020 11:22:01 -0500
Subject: [PATCH 10/67] Remove [client]introducer.furl from test_node

---
 src/allmydata/test/test_node.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/src/allmydata/test/test_node.py b/src/allmydata/test/test_node.py
index 693183ea4..c758262f1 100644
--- a/src/allmydata/test/test_node.py
+++ b/src/allmydata/test/test_node.py
@@ -605,8 +605,6 @@ class TestMissingPorts(unittest.TestCase):
 
 
 BASE_CONFIG = """
-[client]
-introducer.furl = empty
 [tor]
 enabled = false
 [i2p]

From 0f4e34c41da6d9ce5461f219a06d71bb5c218714 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Sat, 14 Nov 2020 11:32:58 -0500
Subject: [PATCH 11/67] Take [client]introducer.furl out of the UseNode fixture

---
 src/allmydata/test/common.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/src/allmydata/test/common.py b/src/allmydata/test/common.py
index 2bf3ca50d..f07a3e74f 100644
--- a/src/allmydata/test/common.py
+++ b/src/allmydata/test/common.py
@@ -267,6 +267,11 @@ class UseNode(object):
     config=format_config_items(self.plugin_config),
 )
 
+        write_introducer(
+            self.basedir.asBytesMode().path,
+            "default",
+            self.introducer_furl,
+        )
         self.config = config_from_string(
             self.basedir.asTextMode().path,
             "tub.port",
@@ -275,11 +280,9 @@ class UseNode(object):
 {node_config}
 
 [client]
-introducer.furl = {furl}
 storage.plugins = {storage_plugin}
 {plugin_config_section}
 """.format(
-    furl=self.introducer_furl,
     storage_plugin=self.storage_plugin,
     node_config=format_config_items(self.node_config),
     plugin_config_section=plugin_config_section,

From 18e327417c740b4f92a0882c9c3d8d2972e3f4d5 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Sat, 14 Nov 2020 11:43:28 -0500
Subject: [PATCH 12/67] Get [client]introducer.furl out of test_system

---
 src/allmydata/test/test_system.py | 25 ++++++++++++++-----------
 1 file changed, 14 insertions(+), 11 deletions(-)

diff --git a/src/allmydata/test/test_system.py b/src/allmydata/test/test_system.py
index 21da0a914..14bcb9c47 100644
--- a/src/allmydata/test/test_system.py
+++ b/src/allmydata/test/test_system.py
@@ -47,6 +47,9 @@ from .web.common import (
 from allmydata.test.test_runner import RunBinTahoeMixin
 from . import common_util as testutil
 from .common_util import run_cli
+from .common import (
+    write_introducer,
+)
 
 LARGE_DATA = """
 This is some data to publish to the remote grid.., which needs to be large
@@ -806,8 +809,6 @@ class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin):
 
         except1 = set(range(self.numclients)) - {1}
         feature_matrix = {
-            # client 1 uses private/introducers.yaml, not tahoe.cfg
-            ("client", "introducer.furl"): except1,
             ("client", "nickname"): except1,
 
             # client 1 has to auto-assign an address.
@@ -833,7 +834,6 @@ class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin):
         setnode = partial(setconf, config, which, "node")
         sethelper = partial(setconf, config, which, "helper")
 
-        setclient("introducer.furl", self.introducer_furl)
         setnode("nickname", u"client %d \N{BLACK SMILING FACE}" % (which,))
 
         if self.stats_gatherer_furl:
@@ -850,13 +850,11 @@ class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin):
 
         sethelper("enabled", "True")
 
-        if which == 1:
-            # clients[1] uses private/introducers.yaml, not tahoe.cfg
-            iyaml = ("introducers:\n"
-                     " petname2:\n"
-                     "  furl: %s\n") % self.introducer_furl
-            iyaml_fn = os.path.join(basedir, "private", "introducers.yaml")
-            fileutil.write(iyaml_fn, iyaml)
+        iyaml = ("introducers:\n"
+                 " petname2:\n"
+                 "  furl: %s\n") % self.introducer_furl
+        iyaml_fn = os.path.join(basedir, "private", "introducers.yaml")
+        fileutil.write(iyaml_fn, iyaml)
 
         return _render_config(config)
 
@@ -909,10 +907,15 @@ class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin):
         if not os.path.isdir(basedir):
             fileutil.make_dirs(basedir)
         config = "[client]\n"
-        config += "introducer.furl = %s\n" % self.introducer_furl
         if helper_furl:
             config += "helper.furl = %s\n" % helper_furl
         fileutil.write(os.path.join(basedir, 'tahoe.cfg'), config)
+        os.makedirs(basedir + b"/private")
+        write_introducer(
+            basedir,
+            "default",
+            self.introducer_furl,
+        )
 
         c = yield client.create_client(basedir)
         self.clients.append(c)

From 22973e6951af6a320473862534743586b173741e Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Sat, 14 Nov 2020 12:36:22 -0500
Subject: [PATCH 13/67] Attempt to make Python 3 happier

---
 src/allmydata/test/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/allmydata/test/common.py b/src/allmydata/test/common.py
index f07a3e74f..4475e287a 100644
--- a/src/allmydata/test/common.py
+++ b/src/allmydata/test/common.py
@@ -116,7 +116,7 @@ def write_introducer(basedir, petname, furl):
                     "furl": furl,
                 },
             },
-        }),
+        }).encode("ascii"),
     )
 
 

From 7b2d76c7ecf0db54990dc482ded4c9c6ef88143d Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Sat, 14 Nov 2020 16:12:07 -0500
Subject: [PATCH 14/67] Another effort to make this simultaneously Py2/Py3
 friendly

---
 src/allmydata/test/common.py              | 2 +-
 src/allmydata/test/test_storage_client.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/allmydata/test/common.py b/src/allmydata/test/common.py
index 4475e287a..c4f937aad 100644
--- a/src/allmydata/test/common.py
+++ b/src/allmydata/test/common.py
@@ -243,7 +243,7 @@ class UseNode(object):
     plugin_config = attr.ib()
     storage_plugin = attr.ib()
     basedir = attr.ib()
-    introducer_furl = attr.ib()
+    introducer_furl = attr.ib(validator=attr.validators.instance_of(bytes))
     node_config = attr.ib(default=attr.Factory(dict))
 
     config = attr.ib(default=None)
diff --git a/src/allmydata/test/test_storage_client.py b/src/allmydata/test/test_storage_client.py
index fa3a34b15..421c588ff 100644
--- a/src/allmydata/test/test_storage_client.py
+++ b/src/allmydata/test/test_storage_client.py
@@ -458,7 +458,7 @@ class StoragePluginWebPresence(AsyncTestCase):
             },
             storage_plugin=self.storage_plugin,
             basedir=self.basedir,
-            introducer_furl=ensure_text(SOME_FURL),
+            introducer_furl=SOME_FURL,
         ))
         self.node = yield self.node_fixture.create_node()
         self.webish = self.node.getServiceNamed(WebishServer.name)

From 06fe3869ef68d4db7208a6b150f4c327afdb2e2b Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Sat, 14 Nov 2020 16:36:51 -0500
Subject: [PATCH 15/67] is pyyaml screwing it up?

---
 src/allmydata/node.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/allmydata/node.py b/src/allmydata/node.py
index 87a9579d8..680e8e1a7 100644
--- a/src/allmydata/node.py
+++ b/src/allmydata/node.py
@@ -463,6 +463,8 @@ class _Config(object):
                     for petname, config
                     in introducers_yaml.get("introducers", {}).items()
                 }
+                assert all(isinstance(unicode, k) for k in introducers.keys())
+                assert all(isinstance(unicode, v) for v in introducers.values())
                 log.msg(
                     "found {} introducers in private/introducers.yaml".format(
                         len(introducers),

From 4963e78f9f7501b3112e9e7f57afb004a11c212d Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Sat, 14 Nov 2020 16:46:46 -0500
Subject: [PATCH 16/67] speed up ci-based testing

---
 .circleci/config.yml | 100 +++++++++++++++++++++----------------------
 1 file changed, 50 insertions(+), 50 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index afa3fafa1..424fc1dd4 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -26,70 +26,70 @@ workflows:
       # Contexts are managed in the CircleCI web interface:
       #
       #  https://app.circleci.com/settings/organization/github/tahoe-lafs/contexts
-      - "debian-9": &DOCKERHUB_CONTEXT
-          context: "dockerhub-auth"
+      # - "debian-9": &DOCKERHUB_CONTEXT
+      #     context: "dockerhub-auth"
 
-      - "debian-8":
-          <<: *DOCKERHUB_CONTEXT
-          requires:
-            - "debian-9"
+      # - "debian-8":
+      #     <<: *DOCKERHUB_CONTEXT
+      #     requires:
+      #       - "debian-9"
 
-      - "ubuntu-20-04":
-          <<: *DOCKERHUB_CONTEXT
-      - "ubuntu-18-04":
-          <<: *DOCKERHUB_CONTEXT
-          requires:
-            - "ubuntu-20-04"
-      - "ubuntu-16-04":
-          <<: *DOCKERHUB_CONTEXT
-          requires:
-            - "ubuntu-20-04"
+      # - "ubuntu-20-04":
+      #     <<: *DOCKERHUB_CONTEXT
+      # - "ubuntu-18-04":
+      #     <<: *DOCKERHUB_CONTEXT
+      #     requires:
+      #       - "ubuntu-20-04"
+      # - "ubuntu-16-04":
+      #     <<: *DOCKERHUB_CONTEXT
+      #     requires:
+      #       - "ubuntu-20-04"
 
-      - "fedora-29":
-          <<: *DOCKERHUB_CONTEXT
-      - "fedora-28":
-          <<: *DOCKERHUB_CONTEXT
-          requires:
-            - "fedora-29"
+      # - "fedora-29":
+      #     <<: *DOCKERHUB_CONTEXT
+      # - "fedora-28":
+      #     <<: *DOCKERHUB_CONTEXT
+      #     requires:
+      #       - "fedora-29"
 
-      - "centos-8":
-          <<: *DOCKERHUB_CONTEXT
+      # - "centos-8":
+      #     <<: *DOCKERHUB_CONTEXT
 
-      - "nixos-19-09":
-          <<: *DOCKERHUB_CONTEXT
+      # - "nixos-19-09":
+      #     <<: *DOCKERHUB_CONTEXT
 
-      # Test against PyPy 2.7
-      - "pypy27-buster":
-          <<: *DOCKERHUB_CONTEXT
+      # # Test against PyPy 2.7
+      # - "pypy27-buster":
+      #     <<: *DOCKERHUB_CONTEXT
 
       # Just one Python 3.6 configuration while the port is in-progress.
       - "python36":
           <<: *DOCKERHUB_CONTEXT
 
       # Other assorted tasks and configurations
-      - "lint":
-          <<: *DOCKERHUB_CONTEXT
-      - "pyinstaller":
-          <<: *DOCKERHUB_CONTEXT
-      - "deprecations":
-          <<: *DOCKERHUB_CONTEXT
-      - "c-locale":
-          <<: *DOCKERHUB_CONTEXT
-      # Any locale other than C or UTF-8.
-      - "another-locale":
-          <<: *DOCKERHUB_CONTEXT
+      # - "lint":
+      #     <<: *DOCKERHUB_CONTEXT
+      # - "pyinstaller":
+      #     <<: *DOCKERHUB_CONTEXT
+      # - "deprecations":
+      #     <<: *DOCKERHUB_CONTEXT
+      # - "c-locale":
+      #     <<: *DOCKERHUB_CONTEXT
+      # # Any locale other than C or UTF-8.
+      # - "another-locale":
+      #     <<: *DOCKERHUB_CONTEXT
 
-      - "integration":
-          <<: *DOCKERHUB_CONTEXT
-          requires:
-            # If the unit test suite doesn't pass, don't bother running the
-            # integration tests.
-            - "debian-9"
+      # - "integration":
+      #     <<: *DOCKERHUB_CONTEXT
+      #     requires:
+      #       # If the unit test suite doesn't pass, don't bother running the
+      #       # integration tests.
+      #       - "debian-9"
 
-      # Generate the underlying data for a visualization to aid with Python 3
-      # porting.
-      - "build-porting-depgraph":
-          <<: *DOCKERHUB_CONTEXT
+      # # Generate the underlying data for a visualization to aid with Python 3
+      # # porting.
+      # - "build-porting-depgraph":
+      #     <<: *DOCKERHUB_CONTEXT
 
   images:
     # Build the Docker images used by the ci jobs.  This makes the ci jobs

From c529d271eedf3438739c2d1fdd98fc12949f0e79 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Sat, 14 Nov 2020 16:46:54 -0500
Subject: [PATCH 17/67] "unicode" is spelled "str" now

---
 src/allmydata/node.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/allmydata/node.py b/src/allmydata/node.py
index 680e8e1a7..83c6d23a9 100644
--- a/src/allmydata/node.py
+++ b/src/allmydata/node.py
@@ -463,8 +463,8 @@ class _Config(object):
                     for petname, config
                     in introducers_yaml.get("introducers", {}).items()
                 }
-                assert all(isinstance(unicode, k) for k in introducers.keys())
-                assert all(isinstance(unicode, v) for v in introducers.values())
+                assert all(isinstance(str, k) for k in introducers.keys())
+                assert all(isinstance(str, v) for v in introducers.values())
                 log.msg(
                     "found {} introducers in private/introducers.yaml".format(
                         len(introducers),

From fd495124efcf63b7f9629a026a5e05d93cebcca2 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Sat, 14 Nov 2020 16:47:52 -0500
Subject: [PATCH 18/67] bleh can't just comment it all out anymore

---
 .circleci/config.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 424fc1dd4..0149d3768 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -26,8 +26,8 @@ workflows:
       # Contexts are managed in the CircleCI web interface:
       #
       #  https://app.circleci.com/settings/organization/github/tahoe-lafs/contexts
-      # - "debian-9": &DOCKERHUB_CONTEXT
-      #     context: "dockerhub-auth"
+      - "debian-9": &DOCKERHUB_CONTEXT
+          context: "dockerhub-auth"
 
       # - "debian-8":
       #     <<: *DOCKERHUB_CONTEXT

From dbb8050a8c2e2439b04ad27e5f6de0ff7501d27b Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Sat, 14 Nov 2020 16:51:36 -0500
Subject: [PATCH 19/67] really suffering from not having a local dev env here

---
 src/allmydata/node.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/allmydata/node.py b/src/allmydata/node.py
index 83c6d23a9..56b0c39f0 100644
--- a/src/allmydata/node.py
+++ b/src/allmydata/node.py
@@ -463,8 +463,8 @@ class _Config(object):
                     for petname, config
                     in introducers_yaml.get("introducers", {}).items()
                 }
-                assert all(isinstance(str, k) for k in introducers.keys())
-                assert all(isinstance(str, v) for v in introducers.values())
+                assert all(isinstance(k, str) for k in introducers.keys())
+                assert all(isinstance(v, str) for v in introducers.values())
                 log.msg(
                     "found {} introducers in private/introducers.yaml".format(
                         len(introducers),

From 3ac2e9365fc240be0ec7dbbfddd1902f7acd95b8 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Sat, 14 Nov 2020 16:57:45 -0500
Subject: [PATCH 20/67] yea okay that one fails

---
 src/allmydata/node.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/allmydata/node.py b/src/allmydata/node.py
index 56b0c39f0..f467f95b9 100644
--- a/src/allmydata/node.py
+++ b/src/allmydata/node.py
@@ -464,7 +464,7 @@ class _Config(object):
                     in introducers_yaml.get("introducers", {}).items()
                 }
                 assert all(isinstance(k, str) for k in introducers.keys())
-                assert all(isinstance(v, str) for v in introducers.values())
+                assert all(isinstance(v, str) for v in introducers.values()), introducers.values()
                 log.msg(
                     "found {} introducers in private/introducers.yaml".format(
                         len(introducers),

From d27c25a26f290dc71be03eba22432dd839f8ae2c Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Sat, 14 Nov 2020 18:17:34 -0500
Subject: [PATCH 21/67] make sure we put text into yaml

---
 src/allmydata/test/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/allmydata/test/common.py b/src/allmydata/test/common.py
index c4f937aad..8eab155b0 100644
--- a/src/allmydata/test/common.py
+++ b/src/allmydata/test/common.py
@@ -113,7 +113,7 @@ def write_introducer(basedir, petname, furl):
         safe_dump({
             "introducers": {
                 petname: {
-                    "furl": furl,
+                    "furl": furl.decode("ascii"),
                 },
             },
         }).encode("ascii"),

From 7cf5b04b770713b0f4eb103d368385661b1feb34 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Sat, 14 Nov 2020 18:22:41 -0500
Subject: [PATCH 22/67] Put all the CI back

---
 .circleci/config.yml | 96 ++++++++++++++++++++++----------------------
 1 file changed, 48 insertions(+), 48 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 0149d3768..afa3fafa1 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -29,67 +29,67 @@ workflows:
       - "debian-9": &DOCKERHUB_CONTEXT
           context: "dockerhub-auth"
 
-      # - "debian-8":
-      #     <<: *DOCKERHUB_CONTEXT
-      #     requires:
-      #       - "debian-9"
+      - "debian-8":
+          <<: *DOCKERHUB_CONTEXT
+          requires:
+            - "debian-9"
 
-      # - "ubuntu-20-04":
-      #     <<: *DOCKERHUB_CONTEXT
-      # - "ubuntu-18-04":
-      #     <<: *DOCKERHUB_CONTEXT
-      #     requires:
-      #       - "ubuntu-20-04"
-      # - "ubuntu-16-04":
-      #     <<: *DOCKERHUB_CONTEXT
-      #     requires:
-      #       - "ubuntu-20-04"
+      - "ubuntu-20-04":
+          <<: *DOCKERHUB_CONTEXT
+      - "ubuntu-18-04":
+          <<: *DOCKERHUB_CONTEXT
+          requires:
+            - "ubuntu-20-04"
+      - "ubuntu-16-04":
+          <<: *DOCKERHUB_CONTEXT
+          requires:
+            - "ubuntu-20-04"
 
-      # - "fedora-29":
-      #     <<: *DOCKERHUB_CONTEXT
-      # - "fedora-28":
-      #     <<: *DOCKERHUB_CONTEXT
-      #     requires:
-      #       - "fedora-29"
+      - "fedora-29":
+          <<: *DOCKERHUB_CONTEXT
+      - "fedora-28":
+          <<: *DOCKERHUB_CONTEXT
+          requires:
+            - "fedora-29"
 
-      # - "centos-8":
-      #     <<: *DOCKERHUB_CONTEXT
+      - "centos-8":
+          <<: *DOCKERHUB_CONTEXT
 
-      # - "nixos-19-09":
-      #     <<: *DOCKERHUB_CONTEXT
+      - "nixos-19-09":
+          <<: *DOCKERHUB_CONTEXT
 
-      # # Test against PyPy 2.7
-      # - "pypy27-buster":
-      #     <<: *DOCKERHUB_CONTEXT
+      # Test against PyPy 2.7
+      - "pypy27-buster":
+          <<: *DOCKERHUB_CONTEXT
 
       # Just one Python 3.6 configuration while the port is in-progress.
       - "python36":
           <<: *DOCKERHUB_CONTEXT
 
       # Other assorted tasks and configurations
-      # - "lint":
-      #     <<: *DOCKERHUB_CONTEXT
-      # - "pyinstaller":
-      #     <<: *DOCKERHUB_CONTEXT
-      # - "deprecations":
-      #     <<: *DOCKERHUB_CONTEXT
-      # - "c-locale":
-      #     <<: *DOCKERHUB_CONTEXT
-      # # Any locale other than C or UTF-8.
-      # - "another-locale":
-      #     <<: *DOCKERHUB_CONTEXT
+      - "lint":
+          <<: *DOCKERHUB_CONTEXT
+      - "pyinstaller":
+          <<: *DOCKERHUB_CONTEXT
+      - "deprecations":
+          <<: *DOCKERHUB_CONTEXT
+      - "c-locale":
+          <<: *DOCKERHUB_CONTEXT
+      # Any locale other than C or UTF-8.
+      - "another-locale":
+          <<: *DOCKERHUB_CONTEXT
 
-      # - "integration":
-      #     <<: *DOCKERHUB_CONTEXT
-      #     requires:
-      #       # If the unit test suite doesn't pass, don't bother running the
-      #       # integration tests.
-      #       - "debian-9"
+      - "integration":
+          <<: *DOCKERHUB_CONTEXT
+          requires:
+            # If the unit test suite doesn't pass, don't bother running the
+            # integration tests.
+            - "debian-9"
 
-      # # Generate the underlying data for a visualization to aid with Python 3
-      # # porting.
-      # - "build-porting-depgraph":
-      #     <<: *DOCKERHUB_CONTEXT
+      # Generate the underlying data for a visualization to aid with Python 3
+      # porting.
+      - "build-porting-depgraph":
+          <<: *DOCKERHUB_CONTEXT
 
   images:
     # Build the Docker images used by the ci jobs.  This makes the ci jobs

From 10600ef5ec1c138d7d852413f9f79e01350b540a Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Mon, 16 Nov 2020 14:59:42 -0500
Subject: [PATCH 23/67] Move write_introducer somewhere it can be used more
 widely

---
 src/allmydata/scripts/common.py       | 26 +++++++++++++++++++++++---
 src/allmydata/test/common.py          | 23 +++--------------------
 src/allmydata/test/test_client.py     |  4 +++-
 src/allmydata/test/test_introducer.py |  4 +++-
 src/allmydata/test/test_system.py     |  2 +-
 5 files changed, 33 insertions(+), 26 deletions(-)

diff --git a/src/allmydata/scripts/common.py b/src/allmydata/scripts/common.py
index 34266ee72..572048f71 100644
--- a/src/allmydata/scripts/common.py
+++ b/src/allmydata/scripts/common.py
@@ -4,15 +4,19 @@ import os, sys, urllib, textwrap
 import codecs
 from os.path import join
 
+from yaml import (
+    safe_dump,
+)
+
 # Python 2 compatibility
 from future.utils import PY2
 if PY2:
     from future.builtins import str  # noqa: F401
 
-# On Python 2 this will be the backported package:
-from configparser import NoSectionError
-
 from twisted.python import usage
+from twisted.python.filepath import FilePath
+
+
 
 from allmydata.util.assertutil import precondition
 from allmydata.util.encodingutil import unicode_to_url, quote_output, \
@@ -115,6 +119,22 @@ class NoDefaultBasedirOptions(BasedirOptions):
 DEFAULT_ALIAS = u"tahoe"
 
 
+def write_introducer(basedir, petname, furl):
+    """
+    Overwrite the node's ``introducers.yaml`` with a file containing the given
+    introducer information.
+    """
+    FilePath(basedir).child(b"private").child(b"introducers.yaml").setContent(
+        safe_dump({
+            "introducers": {
+                petname: {
+                    "furl": furl.decode("ascii"),
+                },
+            },
+        }).encode("ascii"),
+    )
+
+
 def get_introducer_furl(nodedir, config):
     """
     :return: the introducer FURL for the given node (no matter if it's
diff --git a/src/allmydata/test/common.py b/src/allmydata/test/common.py
index 8eab155b0..cbe4d768c 100644
--- a/src/allmydata/test/common.py
+++ b/src/allmydata/test/common.py
@@ -32,10 +32,6 @@ import attr
 
 import treq
 
-from yaml import (
-    safe_dump,
-)
-
 from zope.interface import implementer
 
 from testtools import (
@@ -85,6 +81,9 @@ from allmydata.client import (
     config_from_string,
     create_client_from_config,
 )
+from allmydata.scripts.common import (
+    write_introducer,
+    )
 
 from ..crypto import (
     ed25519,
@@ -104,22 +103,6 @@ EMPTY_CLIENT_CONFIG = config_from_string(
 )
 
 
-def write_introducer(basedir, petname, furl):
-    """
-    Overwrite the node's ``introducers.yaml`` with a file containing the given
-    introducer information.
-    """
-    FilePath(basedir).child(b"private").child(b"introducers.yaml").setContent(
-        safe_dump({
-            "introducers": {
-                petname: {
-                    "furl": furl.decode("ascii"),
-                },
-            },
-        }).encode("ascii"),
-    )
-
-
 @attr.s
 class MemoryIntroducerClient(object):
     """
diff --git a/src/allmydata/test/test_client.py b/src/allmydata/test/test_client.py
index b76047be0..7fb82c262 100644
--- a/src/allmydata/test/test_client.py
+++ b/src/allmydata/test/test_client.py
@@ -58,6 +58,9 @@ from allmydata.util import (
 from allmydata.util.fileutil import abspath_expanduser_unicode
 from allmydata.interfaces import IFilesystemNode, IFileNode, \
      IImmutableFileNode, IMutableFileNode, IDirectoryNode
+from allmydata.scripts.common import (
+    write_introducer,
+)
 from foolscap.api import flushEventualQueue
 import allmydata.test.common_util as testutil
 from .common import (
@@ -66,7 +69,6 @@ from .common import (
     UseTestPlugins,
     MemoryIntroducerClient,
     get_published_announcements,
-    write_introducer,
 )
 from .matchers import (
     MatchesSameElements,
diff --git a/src/allmydata/test/test_introducer.py b/src/allmydata/test/test_introducer.py
index 4ccabd4e4..f6497eeb4 100644
--- a/src/allmydata/test/test_introducer.py
+++ b/src/allmydata/test/test_introducer.py
@@ -39,9 +39,11 @@ from allmydata.util import pollmixin, idlib, fileutil, yamlutil
 from allmydata.util.iputil import (
     listenOnUnused,
 )
+from allmydata.scripts.common import (
+    write_introducer,
+)
 import allmydata.test.common_util as testutil
 from .common import (
-    write_introducer,
     SyncTestCase,
     AsyncTestCase,
     AsyncBrokenTestCase,
diff --git a/src/allmydata/test/test_system.py b/src/allmydata/test/test_system.py
index 14bcb9c47..5a34dcc4d 100644
--- a/src/allmydata/test/test_system.py
+++ b/src/allmydata/test/test_system.py
@@ -47,7 +47,7 @@ from .web.common import (
 from allmydata.test.test_runner import RunBinTahoeMixin
 from . import common_util as testutil
 from .common_util import run_cli
-from .common import (
+from ..scripts.common import (
     write_introducer,
 )
 

From 4e84f5e690b997becba221e5d8dbe8dac2bf7075 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Mon, 16 Nov 2020 15:00:20 -0500
Subject: [PATCH 24/67] write introducers.yaml instead of
 [client]introducer.furl in client creation

---
 src/allmydata/scripts/create_node.py | 29 +++++++++++++++++++++-------
 1 file changed, 22 insertions(+), 7 deletions(-)

diff --git a/src/allmydata/scripts/create_node.py b/src/allmydata/scripts/create_node.py
index 2634e0915..89d81722e 100644
--- a/src/allmydata/scripts/create_node.py
+++ b/src/allmydata/scripts/create_node.py
@@ -3,13 +3,22 @@ from __future__ import print_function
 import os
 import json
 
+from twisted.python.filepath import (
+    FilePath,
+)
 from twisted.internet import reactor, defer
 from twisted.python.usage import UsageError
-from allmydata.scripts.common import BasedirOptions, NoDefaultBasedirOptions
+
+from allmydata.scripts.common import (
+    BasedirOptions,
+    NoDefaultBasedirOptions,
+    write_introducer,
+)
 from allmydata.scripts.default_nodedir import _default_nodedir
 from allmydata.util.assertutil import precondition
 from allmydata.util.encodingutil import listdir_unicode, argv_to_unicode, quote_local_unicode_path, get_io_encoding
 from allmydata.util import fileutil, i2p_provider, iputil, tor_provider
+
 from wormhole import wormhole
 
 
@@ -299,12 +308,15 @@ def write_node_config(c, config):
 
 
 def write_client_config(c, config):
-    # note, config can be a plain dict, it seems -- see
-    # test_configutil.py in test_create_client_config
+    introducer = config.get("introducer", None)
+    if introducer is not None:
+        write_introducer(
+            config["basedir"],
+            "default",
+            introducer,
+        )
+
     c.write("[client]\n")
-    c.write("# Which services should this client connect to?\n")
-    introducer = config.get("introducer", None) or ""
-    c.write("introducer.furl = %s\n" % introducer)
     c.write("helper.furl =\n")
     c.write("#stats_gatherer.furl =\n")
     c.write("\n")
@@ -437,8 +449,11 @@ def create_node(config):
 
     print("Node created in %s" % quote_local_unicode_path(basedir), file=out)
     tahoe_cfg = quote_local_unicode_path(os.path.join(basedir, "tahoe.cfg"))
+    introducers_yaml = quote_local_unicode_path(
+        os.path.join(basedir, "private", "introducers.yaml"),
+    )
     if not config.get("introducer", ""):
-        print(" Please set [client]introducer.furl= in %s!" % tahoe_cfg, file=out)
+        print(" Please add introducers to %s!" % (introducers_yaml,), file=out)
         print(" The node cannot connect to a grid without it.", file=out)
     if not config.get("nickname", ""):
         print(" Please set [node]nickname= in %s" % tahoe_cfg, file=out)

From 1946ee502360eaf50d3202f912e38b0acd998d0c Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Mon, 16 Nov 2020 15:00:49 -0500
Subject: [PATCH 25/67] note this is for deprecated functionality

---
 src/allmydata/test/test_client.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/allmydata/test/test_client.py b/src/allmydata/test/test_client.py
index 7fb82c262..02423a475 100644
--- a/src/allmydata/test/test_client.py
+++ b/src/allmydata/test/test_client.py
@@ -786,8 +786,8 @@ class IntroducerClients(unittest.TestCase):
 
     def test_invalid_introducer_furl(self):
         """
-        An introducer.furl of 'None' is invalid and causes
-        create_introducer_clients to fail.
+        An introducer.furl of 'None' in the deprecated [client]introducer.furl
+        field is invalid and causes `create_introducer_clients` to fail.
         """
         cfg = (
             "[client]\n"

From 69b8262f6b196bf83e6767cd92f996bb669ec53d Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Mon, 16 Nov 2020 15:01:05 -0500
Subject: [PATCH 26/67] use a different .furl item since introducer.furl will
 go away

---
 src/allmydata/test/test_client.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/allmydata/test/test_client.py b/src/allmydata/test/test_client.py
index 02423a475..e082a33d1 100644
--- a/src/allmydata/test/test_client.py
+++ b/src/allmydata/test/test_client.py
@@ -120,14 +120,14 @@ class Basic(testutil.ReallyEqualMixin, unittest.TestCase):
 
         def write_config(s):
             config = ("[client]\n"
-                      "introducer.furl = %s\n" % s)
+                      "helper.furl = %s\n" % s)
             fileutil.write(os.path.join(basedir, "tahoe.cfg"), config)
 
         for s in should_fail:
             write_config(s)
             with self.assertRaises(UnescapedHashError) as ctx:
                 yield client.create_client(basedir)
-            self.assertIn("[client]introducer.furl", str(ctx.exception))
+            self.assertIn("[client]helper.furl", str(ctx.exception))
 
     def test_unreadable_config(self):
         if sys.platform == "win32":

From 0fd354396f207d071788a370379273c7fd61292e Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Mon, 16 Nov 2020 15:01:21 -0500
Subject: [PATCH 27/67] note this is for deprecated functionality

---
 src/allmydata/test/test_multi_introducers.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/allmydata/test/test_multi_introducers.py b/src/allmydata/test/test_multi_introducers.py
index 36801c740..520a5a69a 100644
--- a/src/allmydata/test/test_multi_introducers.py
+++ b/src/allmydata/test/test_multi_introducers.py
@@ -88,7 +88,8 @@ class MultiIntroTests(unittest.TestCase):
     @defer.inlineCallbacks
     def test_reject_default_in_yaml(self):
         """
-        If an introducer is configured in tahoe.cfg then a "default" introducer in
+        If an introducer is configured in tahoe.cfg with the deprecated
+        [client]introducer.furl then a "default" introducer in
         introducers.yaml is rejected.
         """
         connections = {

From 302b5cb01fc0c3bf48e90aa53d1dd5c423754b10 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Mon, 16 Nov 2020 15:01:34 -0500
Subject: [PATCH 28/67] look for the introducer furl via a more structured
 interface

---
 src/allmydata/test/cli/test_invite.py | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

diff --git a/src/allmydata/test/cli/test_invite.py b/src/allmydata/test/cli/test_invite.py
index 0daeb5840..f356e18de 100644
--- a/src/allmydata/test/cli/test_invite.py
+++ b/src/allmydata/test/cli/test_invite.py
@@ -8,7 +8,9 @@ from twisted.internet import defer
 from ..common_util import run_cli
 from ..no_network import GridTestMixin
 from .common import CLITestMixin
-
+from ...client import (
+    read_config,
+)
 
 class _FakeWormhole(object):
 
@@ -81,9 +83,19 @@ class Join(GridTestMixin, CLITestMixin, unittest.TestCase):
             )
 
             self.assertEqual(0, rc)
+
+            config = read_config(node_dir, u"")
+            self.assertIn(
+                "pb://foo",
+                set(
+                    furl
+                    for (furl, cache)
+                    in config.get_introducer_configuration().values()
+                ),
+            )
+
             with open(join(node_dir, 'tahoe.cfg'), 'r') as f:
                 config = f.read()
-            self.assertIn("pb://foo", config)
             self.assertIn(u"somethinghopefullyunique", config)
 
     @defer.inlineCallbacks

From c9f7ce8db5ca883f07ce3fa77c1a4b2922d24850 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Mon, 16 Nov 2020 15:01:52 -0500
Subject: [PATCH 29/67] write introducers.yaml instead of
 [client]introducer.furl

---
 src/allmydata/test/check_memory.py | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/src/allmydata/test/check_memory.py b/src/allmydata/test/check_memory.py
index 41cf6e1d7..3ab88b9c6 100644
--- a/src/allmydata/test/check_memory.py
+++ b/src/allmydata/test/check_memory.py
@@ -21,6 +21,10 @@ from allmydata.util import fileutil, pollmixin
 from allmydata.util.fileutil import abspath_expanduser_unicode
 from allmydata.util.encodingutil import get_filesystem_encoding
 
+from allmydata.scripts.common import (
+    write_introducer,
+)
+
 class StallableHTTPGetterDiscarder(tw_client.HTTPPageGetter, object):
     full_speed_ahead = False
     _bytes_so_far = 0
@@ -183,13 +187,13 @@ class SystemFramework(pollmixin.PollMixin):
         self.nodes = []
         for i in range(self.numnodes):
             nodedir = os.path.join(self.testdir, "node%d" % i)
-            os.mkdir(nodedir)
+            os.makedirs(nodedir + b"/private")
+            write_introducer(nodedir, "default", self.introducer_url)
             f = open(os.path.join(nodedir, "tahoe.cfg"), "w")
             f.write("[client]\n"
-                    "introducer.furl = %s\n"
                     "shares.happy = 1\n"
                     "[storage]\n"
-                    % (self.introducer_furl,))
+            )
             # the only tests for which we want the internal nodes to actually
             # retain shares are the ones where somebody's going to download
             # them.
@@ -235,16 +239,16 @@ this file are ignored.
         quiet = StringIO()
         create_node.create_node({'basedir': clientdir}, out=quiet)
         log.msg("DONE MAKING CLIENT")
+        write_introducer(clientdir, "default", self.introducer_furl)
         # now replace tahoe.cfg
         # set webport=0 and then ask the node what port it picked.
         f = open(os.path.join(clientdir, "tahoe.cfg"), "w")
         f.write("[node]\n"
                 "web.port = tcp:0:interface=127.0.0.1\n"
                 "[client]\n"
-                "introducer.furl = %s\n"
                 "shares.happy = 1\n"
                 "[storage]\n"
-                % (self.introducer_furl,))
+        )
 
         if self.mode in ("upload-self", "receive"):
             # accept and store shares, to trigger the memory consumption bugs

From 0258bb72954ebbb7c19bb79a9fc927d8382fb1b7 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Mon, 16 Nov 2020 15:02:13 -0500
Subject: [PATCH 30/67] note it's deprecated

---
 src/allmydata/node.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/allmydata/node.py b/src/allmydata/node.py
index f467f95b9..04ba2f2fe 100644
--- a/src/allmydata/node.py
+++ b/src/allmydata/node.py
@@ -475,7 +475,7 @@ class _Config(object):
                 raise
             introducers = {}
 
-        # read furl from tahoe.cfg
+        # supported the deprecated [client]introducer.furl item in tahoe.cfg
         tahoe_cfg_introducer_furl = self.get_config("client", "introducer.furl", None)
         if tahoe_cfg_introducer_furl == "None":
             raise ValueError(

From 60e0056ad82f0bc43ea4fd0189711a7c184f4245 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Mon, 16 Nov 2020 15:02:18 -0500
Subject: [PATCH 31/67] don't guide folks to the deprecated config item

---
 src/allmydata/node.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/allmydata/node.py b/src/allmydata/node.py
index 04ba2f2fe..7eefbbce0 100644
--- a/src/allmydata/node.py
+++ b/src/allmydata/node.py
@@ -480,8 +480,7 @@ class _Config(object):
         if tahoe_cfg_introducer_furl == "None":
             raise ValueError(
                 "tahoe.cfg has invalid 'introducer.furl = None':"
-                " to disable it, use 'introducer.furl ='"
-                " or omit the key entirely"
+                " to disable it omit the key entirely"
             )
         if tahoe_cfg_introducer_furl:
             warn(

From 92206b907e35801fee77a41eaf4ce776d752c602 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Mon, 16 Nov 2020 15:02:29 -0500
Subject: [PATCH 32/67] write introducers.yaml instead of
 [client]introducer.furl

---
 integration/test_tor.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/integration/test_tor.py b/integration/test_tor.py
index 3d169a88f..8dd05fc3f 100644
--- a/integration/test_tor.py
+++ b/integration/test_tor.py
@@ -9,6 +9,10 @@ import pytest_twisted
 
 import util
 
+from allmydata.test.common import (
+    write_introducer,
+)
+
 # see "conftest.py" for the fixtures (e.g. "tor_network")
 
 # XXX: Integration tests that involve Tor do not run reliably on
@@ -89,6 +93,9 @@ def _create_anonymous_node(reactor, name, control_port, request, temp_dir, flog_
         )
         yield proto.done
 
+
+    # Which services should this client connect to?
+    write_introducer(node_dir, "default", introducer_furl)
     with open(join(node_dir, 'tahoe.cfg'), 'w') as f:
         f.write('''
 [node]
@@ -105,15 +112,12 @@ onion = true
 onion.private_key_file = private/tor_onion.privkey
 
 [client]
-# Which services should this client connect to?
-introducer.furl = %(furl)s
 shares.needed = 1
 shares.happy = 1
 shares.total = 2
 
 ''' % {
     'name': name,
-    'furl': introducer_furl,
     'web_port': web_port,
     'log_furl': flog_gatherer,
     'control_port': control_port,

From 5cb1df06c4bff71f1b526b3bb3824f1b80b739c6 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Mon, 16 Nov 2020 15:02:51 -0500
Subject: [PATCH 33/67] delegate introducer furl lookup to the config object

---
 src/allmydata/scripts/common.py       | 24 ++++++++++++------------
 src/allmydata/scripts/tahoe_invite.py |  3 ++-
 2 files changed, 14 insertions(+), 13 deletions(-)

diff --git a/src/allmydata/scripts/common.py b/src/allmydata/scripts/common.py
index 572048f71..6d633b502 100644
--- a/src/allmydata/scripts/common.py
+++ b/src/allmydata/scripts/common.py
@@ -140,19 +140,19 @@ def get_introducer_furl(nodedir, config):
     :return: the introducer FURL for the given node (no matter if it's
         a client-type node or an introducer itself)
     """
+    for petname, (furl, cache) in config.get_introducer_configuration().items():
+        return furl
+
+    # We have no configured introducers.  Maybe this is running *on* the
+    # introducer?  Let's guess, sure why not.
     try:
-        introducer_furl = config.get('client', 'introducer.furl')
-    except NoSectionError:
-        # we're not a client; maybe this is running *on* the introducer?
-        try:
-            with open(join(nodedir, "private", "introducer.furl"), "r") as f:
-                introducer_furl = f.read().strip()
-        except IOError:
-            raise Exception(
-                "Can't find introducer FURL in tahoe.cfg nor "
-                "{}/private/introducer.furl".format(nodedir)
-            )
-    return introducer_furl
+        with open(join(nodedir, "private", "introducer.furl"), "r") as f:
+            return f.read().strip()
+    except IOError:
+        raise Exception(
+            "Can't find introducer FURL in tahoe.cfg nor "
+            "{}/private/introducer.furl".format(nodedir)
+        )
 
 
 def get_aliases(nodedir):
diff --git a/src/allmydata/scripts/tahoe_invite.py b/src/allmydata/scripts/tahoe_invite.py
index cca4216e3..8096fa3c1 100644
--- a/src/allmydata/scripts/tahoe_invite.py
+++ b/src/allmydata/scripts/tahoe_invite.py
@@ -11,6 +11,7 @@ from wormhole import wormhole
 from allmydata.util import configutil
 from allmydata.util.encodingutil import argv_to_abspath
 from allmydata.scripts.common import get_default_nodedir, get_introducer_furl
+from allmydata.node import read_config
 
 
 class InviteOptions(usage.Options):
@@ -77,7 +78,7 @@ def invite(options):
         basedir = argv_to_abspath(options.parent['node-directory'])
     else:
         basedir = get_default_nodedir()
-    config = configutil.get_config(join(basedir, 'tahoe.cfg'))
+    config = read_config(basedir, u"")
     out = options.stdout
     err = options.stderr
 

From 2ee0b1d3c634d05af00b57c542760e3af50ced73 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Mon, 16 Nov 2020 15:05:04 -0500
Subject: [PATCH 34/67] flake cleanup

---
 src/allmydata/scripts/create_node.py  | 3 ---
 src/allmydata/scripts/tahoe_invite.py | 2 --
 2 files changed, 5 deletions(-)

diff --git a/src/allmydata/scripts/create_node.py b/src/allmydata/scripts/create_node.py
index 89d81722e..30024dfd9 100644
--- a/src/allmydata/scripts/create_node.py
+++ b/src/allmydata/scripts/create_node.py
@@ -3,9 +3,6 @@ from __future__ import print_function
 import os
 import json
 
-from twisted.python.filepath import (
-    FilePath,
-)
 from twisted.internet import reactor, defer
 from twisted.python.usage import UsageError
 
diff --git a/src/allmydata/scripts/tahoe_invite.py b/src/allmydata/scripts/tahoe_invite.py
index 8096fa3c1..dbc84d0ea 100644
--- a/src/allmydata/scripts/tahoe_invite.py
+++ b/src/allmydata/scripts/tahoe_invite.py
@@ -1,14 +1,12 @@
 from __future__ import print_function
 
 import json
-from os.path import join
 
 from twisted.python import usage
 from twisted.internet import defer, reactor
 
 from wormhole import wormhole
 
-from allmydata.util import configutil
 from allmydata.util.encodingutil import argv_to_abspath
 from allmydata.scripts.common import get_default_nodedir, get_introducer_furl
 from allmydata.node import read_config

From 38dd0d1b709d8f2f9b4ee5737c2b0f0b6eb5d9ad Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Thu, 19 Nov 2020 11:12:08 -0500
Subject: [PATCH 35/67] Only run codechecks on changed Python source files

---
 .pre-commit-config.yaml | 11 ++++++-----
 tox.ini                 | 14 +++++++++-----
 2 files changed, 15 insertions(+), 10 deletions(-)

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 76162535a..916b331e4 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -1,9 +1,10 @@
 repos:
-  - repo: local
+  - repo: "local"
     hooks:
-    - id: codechecks
-      name: codechecks
+    - id: "codechecks"
+      name: "codechecks"
       stages: ["push"]
+      language: "system"
+      files: ".py$"
       entry: "tox -e codechecks"
-      language: system
-      pass_filenames: false
+      pass_filenames: true
diff --git a/tox.ini b/tox.ini
index 597270e3a..b95476f58 100644
--- a/tox.ini
+++ b/tox.ini
@@ -95,12 +95,16 @@ setenv =
          #     .decode(getattr(sys.stdout, "encoding", "utf8"))
          # `TypeError: decode() argument 1 must be string, not None`
          PYTHONIOENCODING=utf_8
+
+	 # If no positional arguments are given, try to run the checks on the
+	 # entire codebase, including various pieces of supporting code.
+	 DEFAULT_FILES="src integration static misc setup.py"
 commands =
-         flake8 src integration static misc setup.py
-         python misc/coding_tools/check-umids.py src
-         python misc/coding_tools/check-debugging.py
-         python misc/coding_tools/find-trailing-spaces.py -r src static misc setup.py
-         python misc/coding_tools/check-miscaptures.py
+         flake8 {posargs:{env:DEFAULT_FILES}}
+         python misc/coding_tools/check-umids.py {posargs:{env:DEFAULT_FILES}}
+         python misc/coding_tools/check-debugging.py {posargs:{env:DEFAULT_FILES}}
+         python misc/coding_tools/find-trailing-spaces.py -r {posargs:{env:DEFAULT_FILES}}
+         python misc/coding_tools/check-miscaptures.py {posargs:{env:DEFAULT_FILES}}
 
          # If towncrier.check fails, you forgot to add a towncrier news
          # fragment explaining the change in this branch.  Create one at

From c4a67d6b8caf99aa244ec40e976f153fe5f7a2f5 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Thu, 19 Nov 2020 11:12:21 -0500
Subject: [PATCH 36/67] news fragment

---
 newsfragments/3515.minor | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 create mode 100644 newsfragments/3515.minor

diff --git a/newsfragments/3515.minor b/newsfragments/3515.minor
new file mode 100644
index 000000000..e69de29bb

From 737f1f21e1454b9876493d83980ac40a003a31ac Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Tue, 24 Nov 2020 14:04:35 -0500
Subject: [PATCH 37/67] news fragment

---
 newsfragments/3520.minor | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 create mode 100644 newsfragments/3520.minor

diff --git a/newsfragments/3520.minor b/newsfragments/3520.minor
new file mode 100644
index 000000000..e69de29bb

From 1a5efa5ec9a9c1c5f30b9832725e5ec043d33ec8 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Tue, 24 Nov 2020 14:05:00 -0500
Subject: [PATCH 38/67] Just let the test use the real SFTPServer service

---
 src/allmydata/client.py                      |  2 +-
 src/allmydata/frontends/sftpd.py             |  2 ++
 src/allmydata/test/data/openssh-rsa-2048     | 27 ++++++++++++++++++
 src/allmydata/test/data/openssh-rsa-2048.pub |  1 +
 src/allmydata/test/test_client.py            | 30 ++++++++++++++------
 5 files changed, 52 insertions(+), 10 deletions(-)
 create mode 100644 src/allmydata/test/data/openssh-rsa-2048
 create mode 100644 src/allmydata/test/data/openssh-rsa-2048.pub

diff --git a/src/allmydata/client.py b/src/allmydata/client.py
index a768ba354..47073dbca 100644
--- a/src/allmydata/client.py
+++ b/src/allmydata/client.py
@@ -1071,7 +1071,7 @@ class _Client(node.Node, pollmixin.PollMixin):
             if accountfile:
                 accountfile = self.config.get_config_path(accountfile)
             accounturl = self.config.get_config("sftpd", "accounts.url", None)
-            sftp_portstr = self.config.get_config("sftpd", "port", "8022")
+            sftp_portstr = self.config.get_config("sftpd", "port", "tcp:8022")
             pubkey_file = self.config.get_config("sftpd", "host_pubkey_file")
             privkey_file = self.config.get_config("sftpd", "host_privkey_file")
 
diff --git a/src/allmydata/frontends/sftpd.py b/src/allmydata/frontends/sftpd.py
index db914fa45..b25ac0270 100644
--- a/src/allmydata/frontends/sftpd.py
+++ b/src/allmydata/frontends/sftpd.py
@@ -1975,6 +1975,8 @@ class Dispatcher(object):
 
 
 class SFTPServer(service.MultiService):
+    name = "frontend:sftp"
+
     def __init__(self, client, accountfile, accounturl,
                  sftp_portstr, pubkey_file, privkey_file):
         precondition(isinstance(accountfile, (unicode, NoneType)), accountfile)
diff --git a/src/allmydata/test/data/openssh-rsa-2048 b/src/allmydata/test/data/openssh-rsa-2048
new file mode 100644
index 000000000..9c8669d03
--- /dev/null
+++ b/src/allmydata/test/data/openssh-rsa-2048
@@ -0,0 +1,27 @@
+-----BEGIN OPENSSH PRIVATE KEY-----
+b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAABFwAAAAdzc2gtcn
+NhAAAAAwEAAQAAAQEA8eSX2j8BNsH1yEHJpRrXvRDwG605hzxjhfz7me9Vz2t5mUxzEELn
+PefYJi8bqZnHOaOI2HvFU8E+m2zZemDPh5gry911R9AanTNpaSgkwf1s0gMhVL/IEo0FV3
+pEZvcBevbWZD67narrUy4al1OREolM34VPgsR0FvpAH9dGHfUevFdX/I5LloxKHlXwpbY+
+zuX0ZaaLqE15/NzFAFwdyf8A46Y546dj8b1y9seWgh7c3dVghhJrVmx2b0QNWtBBNt9831
+zR4OMS3d8nFKgMaEZQnAtt9IrVAV7a+KoAzM0XbC/hxlq3CfuMns5UI/pfJnO74ubITU+x
+w0X74v6xOwAAA8gG6fYoBun2KAAAAAdzc2gtcnNhAAABAQDx5JfaPwE2wfXIQcmlGte9EP
+AbrTmHPGOF/PuZ71XPa3mZTHMQQuc959gmLxupmcc5o4jYe8VTwT6bbNl6YM+HmCvL3XVH
+0BqdM2lpKCTB/WzSAyFUv8gSjQVXekRm9wF69tZkPrudqutTLhqXU5ESiUzfhU+CxHQW+k
+Af10Yd9R68V1f8jkuWjEoeVfCltj7O5fRlpouoTXn83MUAXB3J/wDjpjnjp2PxvXL2x5aC
+Htzd1WCGEmtWbHZvRA1a0EE233zfXNHg4xLd3ycUqAxoRlCcC230itUBXtr4qgDMzRdsL+
+HGWrcJ+4yezlQj+l8mc7vi5shNT7HDRfvi/rE7AAAAAwEAAQAAAQBc8ukC/RjbULbAJ79z
+SRhDV2HcULj9ZVAc6XRI13XSyUqlhIHmar7uw8sECTAJAMVUOanY/d56a5RCJxZ+dvrn8K
+pLoSJy4N2JMHs95CYTwOzy2i8RoMwhjLzTu3DTW/DerkD9rjlrwYTBpsKjCYKCa+31KgW+
+ivzM44aGdbNEyO+yHaxdcyEr3OLcRMppgZmwTieFnG053lCP5XyYRQmZ1a78G6WOzpOgbO
+2N6Z1sbEqTMVd3oxFZAbmqA8kE4jLJzRcso/SSK5NDs22JzMfxByJQSlitWzDDvHdWpQpy
+8C6Eu7+48ataLI68VOOXuDWDy9Dck0ev89u7Z4vNLWBhAAAAgAndOZZ0C179Um6sn6gmfM
+0ttXEaSIqYNGRhkoYqn9vvw03bOMbSnqdEJiwFhbE/rWv7PypB5MeY7tRoCyBMWsUYj0pA
+HKSl68diLr5g5EOIRGAWu8e//7T2HgZKOo+VaG1IXgmb7PUoAJ6Tzsmb4jdnYfg+BP/TDd
+e9yCcoiT2fAAAAgQD6T7Kr6ECg0ME8vt/ixsjKdA2zS9SIHyjCMXbdMv1Ok1hkr5rRWbbZ
+jm79fF+a8pOQUg30Qw2JUx7II50akt2xL6zesGDDUcOHD2GE/B6Ftji53G3fwWZCqeQ5sD
+YP25qAWlrqDBGJvF+hkEdlceS8etYJ3XWXjNIYwfR7frQvkQAAAIEA92Pq3FWH63TS3Lqe
+mQjhfNV75tU0AwENG+xlI1g0nQb7Qsdbm6rIg6XqewUfw03Q+/AqPvwG/1mbyVF7jRZ+qw
+cl69yM70c9qY74GHjIIOOcC8Kgv29LQrm/VqVp0Lesn5RA8SIiLcMfyYBTEX8V9VY99Zkd
+v6WwRr4XK1bPRgsAAAAOZXhhcmt1bkBiYXJ5b24BAgMEBQ==
+-----END OPENSSH PRIVATE KEY-----
diff --git a/src/allmydata/test/data/openssh-rsa-2048.pub b/src/allmydata/test/data/openssh-rsa-2048.pub
new file mode 100644
index 000000000..3c7a32df6
--- /dev/null
+++ b/src/allmydata/test/data/openssh-rsa-2048.pub
@@ -0,0 +1 @@
+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDx5JfaPwE2wfXIQcmlGte9EPAbrTmHPGOF/PuZ71XPa3mZTHMQQuc959gmLxupmcc5o4jYe8VTwT6bbNl6YM+HmCvL3XVH0BqdM2lpKCTB/WzSAyFUv8gSjQVXekRm9wF69tZkPrudqutTLhqXU5ESiUzfhU+CxHQW+kAf10Yd9R68V1f8jkuWjEoeVfCltj7O5fRlpouoTXn83MUAXB3J/wDjpjnjp2PxvXL2x5aCHtzd1WCGEmtWbHZvRA1a0EE233zfXNHg4xLd3ycUqAxoRlCcC230itUBXtr4qgDMzRdsL+HGWrcJ+4yezlQj+l8mc7vi5shNT7HDRfvi/rE7 exarkun@baryon
diff --git a/src/allmydata/test/test_client.py b/src/allmydata/test/test_client.py
index 0f0648a4c..f6d3d4b84 100644
--- a/src/allmydata/test/test_client.py
+++ b/src/allmydata/test/test_client.py
@@ -1,5 +1,4 @@
 import os, sys
-import mock
 from functools import (
     partial,
 )
@@ -422,19 +421,32 @@ class Basic(testutil.ReallyEqualMixin, unittest.TestCase):
         """
         configuration for sftpd results in it being started
         """
+        root = FilePath(self.mktemp())
+        root.makedirs()
+        accounts = root.child(b"sftp-accounts")
+        accounts.touch()
+
+        data = FilePath(__file__).sibling(b"data")
+        privkey = data.child(b"openssh-rsa-2048")
+        pubkey = data.child(b"openssh-rsa-2048.pub")
+
         basedir = u"client.Basic.test_ftp_create"
         create_node_dir(basedir, "testing")
         with open(os.path.join(basedir, "tahoe.cfg"), "w") as f:
-            f.write(
+            f.write((
                 '[sftpd]\n'
                 'enabled = true\n'
-                'accounts.file = foo\n'
-                'host_pubkey_file = pubkey\n'
-                'host_privkey_file = privkey\n'
-            )
-        with mock.patch('allmydata.frontends.sftpd.SFTPServer') as p:
-            yield client.create_client(basedir)
-        self.assertTrue(p.called)
+                'accounts.file = {}\n'
+                'host_pubkey_file = {}\n'
+                'host_privkey_file = {}\n'
+            ).format(accounts.path, pubkey.path, privkey.path))
+
+        client_node = yield client.create_client(
+            basedir,
+        )
+        sftp = client_node.getServiceNamed("frontend:sftp")
+        self.assertIs(sftp.parent, client_node)
+
 
     @defer.inlineCallbacks
     def test_ftp_auth_keyfile(self):

From c3aff634a0a61c4cc4767f61c6fefb0a9f8063f1 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Tue, 24 Nov 2020 14:17:52 -0500
Subject: [PATCH 39/67] Add missing attribute docs

---
 src/allmydata/storage_client.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py
index 2b0ec5af1..823c15b7a 100644
--- a/src/allmydata/storage_client.py
+++ b/src/allmydata/storage_client.py
@@ -154,6 +154,9 @@ class StorageFarmBroker(service.MultiService):
     I'm also responsible for subscribing to the IntroducerClient to find out
     about new servers as they are announced by the Introducer.
 
+    :ivar _tub_maker: A one-argument callable which accepts a dictionary of
+        "handler overrides" and returns a ``foolscap.api.Tub``.
+
     :ivar StorageClientConfig storage_client_config: Values from the node
         configuration file relating to storage behavior.
     """

From 34b5068f5c7880f1e892394615a21a9d8fa9a4d7 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Tue, 24 Nov 2020 14:18:04 -0500
Subject: [PATCH 40/67] Just use a real Tub for this case

---
 src/allmydata/test/test_storage_client.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/allmydata/test/test_storage_client.py b/src/allmydata/test/test_storage_client.py
index fa3a34b15..3086e7491 100644
--- a/src/allmydata/test/test_storage_client.py
+++ b/src/allmydata/test/test_storage_client.py
@@ -505,11 +505,13 @@ class StoragePluginWebPresence(AsyncTestCase):
         )
 
 
-def make_broker(tub_maker=lambda h: Mock()):
+def make_broker(tub_maker=None):
     """
     Create a ``StorageFarmBroker`` with the given tub maker and an empty
     client configuration.
     """
+    if tub_maker is None:
+        tub_maker = lambda handler_overrides: Tub()
     return StorageFarmBroker(True, tub_maker, EMPTY_CLIENT_CONFIG)
 
 

From 9f4be1fbf03b2a09816ce71ad449479f72d2add7 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Tue, 24 Nov 2020 14:23:20 -0500
Subject: [PATCH 41/67] Speed the tests back up

---
 src/allmydata/test/test_storage_client.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/src/allmydata/test/test_storage_client.py b/src/allmydata/test/test_storage_client.py
index 3086e7491..524d88272 100644
--- a/src/allmydata/test/test_storage_client.py
+++ b/src/allmydata/test/test_storage_client.py
@@ -511,7 +511,13 @@ def make_broker(tub_maker=None):
     client configuration.
     """
     if tub_maker is None:
-        tub_maker = lambda handler_overrides: Tub()
+        # Use a pre-generated key so the tests don't spend a lot of time
+        # generating new ones.
+        data = FilePath(__file__).sibling(b"data")
+        privkey = data.child(b"node.pem")
+        tub_maker = lambda handler_overrides: Tub(
+            certData=privkey.getContent(),
+        )
     return StorageFarmBroker(True, tub_maker, EMPTY_CLIENT_CONFIG)
 
 

From a78c8056fb22f259fd15719a684fca305400dc65 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Tue, 24 Nov 2020 14:51:51 -0500
Subject: [PATCH 42/67] Get rid of the first mock use in
 `test_threshold_reached`

---
 src/allmydata/test/data/node.pem          | 45 +++++++++++++++++++++
 src/allmydata/test/test_storage_client.py | 48 +++++++++++++++++------
 2 files changed, 81 insertions(+), 12 deletions(-)
 create mode 100644 src/allmydata/test/data/node.pem

diff --git a/src/allmydata/test/data/node.pem b/src/allmydata/test/data/node.pem
new file mode 100644
index 000000000..dd0928cdd
--- /dev/null
+++ b/src/allmydata/test/data/node.pem
@@ -0,0 +1,45 @@
+-----BEGIN CERTIFICATE-----
+MIICojCCAYoCAQEwDQYJKoZIhvcNAQELBQAwFzEVMBMGA1UEAwwMbmV3cGJfdGhp
+bmd5MB4XDTIwMDEwMjAxNDAzM1oXDTIxMDEwMTAxNDAzM1owFzEVMBMGA1UEAwwM
+bmV3cGJfdGhpbmd5MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA1iNV
+z07PYwZwucl87QlL2TFZvDxD4flZ/p3BZE3DCT5Efn9w2NT4sHXL1e+R/qsDFuNG
+bw1y1TRM0DGK6Wr0XRT2mLQULNgB8y/HrhcSdONsYRyWdj+LimyECKjwh0iSkApv
+Yj/7IOuq6dOoh67YXPdf75OHLShm4+8q8fuwhBL+nuuO4NhZDJKupYHcnuCkcF88
+LN77HKrrgbpyVmeghUkwJMLeJCewvYVlambgWRiuGGexFgAm6laS3rWetOcdm9eg
+FoA9PKNN6xvPatbj99MPoLpBbzsI64M0yT/wTSw1pj/Nom3rwfMa2OH8Kk7c8R/r
+U3xj4ZY1DTlGERvejQIDAQABMA0GCSqGSIb3DQEBCwUAA4IBAQAwyQjQ3ZgtJ3JW
+r3/EPdqSUBamTfXIpOh9rXmRjPpbe+MvenqIzl4q+GnkL5mdEb1e1hdKQZgFQ5Q5
+tbcNIz6h5C07KaNtbqhZCx5c/RUEH87VeXuAuOqZHbZWJ18q0tnk+YgWER2TOkgE
+RI2AslcsJBt88UUOjHX6/7J3KjPFaAjW1QV3TTsHxk14aYDYJwPdz+ijchgbOPQ0
+i+ilhzcB+qQnOC1s4xQSFo+zblTO7EgqM9KpupYfOVFh46P1Mak2W8EDvhz0livl
+OROXJ6nR/13lmQdfVX6T45d+ITBwtmW2nGAh3oI3JlArGKHaW+7qnuHR72q9FSES
+cEYA/wmk
+-----END CERTIFICATE-----
+-----BEGIN PRIVATE KEY-----
+MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDWI1XPTs9jBnC5
+yXztCUvZMVm8PEPh+Vn+ncFkTcMJPkR+f3DY1PiwdcvV75H+qwMW40ZvDXLVNEzQ
+MYrpavRdFPaYtBQs2AHzL8euFxJ042xhHJZ2P4uKbIQIqPCHSJKQCm9iP/sg66rp
+06iHrthc91/vk4ctKGbj7yrx+7CEEv6e647g2FkMkq6lgdye4KRwXzws3vscquuB
+unJWZ6CFSTAkwt4kJ7C9hWVqZuBZGK4YZ7EWACbqVpLetZ605x2b16AWgD08o03r
+G89q1uP30w+gukFvOwjrgzTJP/BNLDWmP82ibevB8xrY4fwqTtzxH+tTfGPhljUN
+OUYRG96NAgMBAAECggEAJ5xztBx0+nFnisZ9yG8uy6d4XPyc5gE1J4dRDdfgmyYc
+j3XNjx6ePi4cHZ/qVryVnrc+AS7wrgW1q9FuS81QFKPbFdZB4SW3/p85BbgY3uxu
+0Ovz3T3V9y4polx12eCP0/tKLVd+gdF2VTik9Sxfs5rC8VNN7wmJNuK4A/k15sgy
+BIu/R8NlMNGQySNhtccp+dzB8uTyKx5zFZhVvnAK/3YX9BC2V4QBW9JxO4S8N0/9
+48e9Sw/fGCfQ/EFPKGCvTvfuRqJ+4t5k10FygXJ+s+y70ifYi+aSsjJBuranbLJp
+g5TwhuKnTWs8Nth3YRLbcJL4VBIOehjAWy8pDMMtlQKBgQD0O8cHb8cOTGW0BijC
+NDofhA2GooQUUR3WL324PXWZq0DXuBDQhJVBKWO3AYonivhhd/qWO8lea9MEmU41
+nKZ7maS4B8AJLJC08P8GL1uCIE/ezEXEi9JwC1zJiyl595Ap4lSAozH0DwjNvmGL
+5mIdYg0BliqFXbloNJkNlb7INwKBgQDgdGEIWXc5Y1ncWNs6iDIV/t2MlL8vLrP0
+hpkl/QiMndOQyD6JBo0+ZqvOQTSS4NTSxBROjPxvFbEJ3eH8Pmn8gHOf46fzP1OJ
+wlYv0gYzkN4FE/tN6JnO2u9pN0euyyZLM1fnEcrMWColMN8JlWjtA7Gbxm8lkfa4
+3vicaJtlWwKBgQCQYL4ZgVR0+Wit8W4qz+EEPHYafvwBXqp6sXxqa7qXawtb+q3F
+9nqdGLCfwMNA+QA37ksugI1byfXmpBH902r/aiZbvAkj4zpwHH9F0r0PwbY1iSA9
+PkLahX0Gj8OnHFgWynsVyGOBWVnk9oSHxVt+7zWtGG5uhKdUGLPZugocJQKBgB61
+7bzduOFiRZ5PjhdxISE/UQL2Kz6Cbl7rt7Kp72yF/7eUnnHTMqoyFBnRdCcQmi4I
+ZBrnUXbFigamlFAWHhxNWwSqeoVeychUjcRXQT/291nMhRsA02KpNA66YJV6+E9b
+xBA6r/vLqGCUUkAWcFfVpIyC1xxV32MmJvAHpBN3AoGAPF3MUFiO0iKNZfst6Tm3
+rzrldLawDo98DRZ7Yb2kWlWZYqUk/Nvryvo2cns75WGSMDYVbbRp+BY7kZmNYa9K
+iQzKDL54ZRu6V+getJdeAO8yXoCmnZKxt5OHvOSrQMfAmFKSwLwxBbZBfXEyuune
+yfusXLtCgajpreoVIa0xWdQ=
+-----END PRIVATE KEY-----
diff --git a/src/allmydata/test/test_storage_client.py b/src/allmydata/test/test_storage_client.py
index 524d88272..ed1dc30a9 100644
--- a/src/allmydata/test/test_storage_client.py
+++ b/src/allmydata/test/test_storage_client.py
@@ -65,6 +65,7 @@ from .common import (
     UseTestPlugins,
     UseNode,
     SameProcessStreamEndpointAssigner,
+    MemoryIntroducerClient,
 )
 from .common_web import (
     do_http,
@@ -505,19 +506,26 @@ class StoragePluginWebPresence(AsyncTestCase):
         )
 
 
+def new_tub():
+    """
+    Make a new ``Tub`` with a hard-coded private key.
+    """
+    # Use a pre-generated key so the tests don't spend a lot of time
+    # generating new ones.
+    data = FilePath(__file__).sibling(b"data")
+    privkey = data.child(b"node.pem")
+    return Tub(
+        certData=privkey.getContent(),
+    )
+
+
 def make_broker(tub_maker=None):
     """
     Create a ``StorageFarmBroker`` with the given tub maker and an empty
     client configuration.
     """
     if tub_maker is None:
-        # Use a pre-generated key so the tests don't spend a lot of time
-        # generating new ones.
-        data = FilePath(__file__).sibling(b"data")
-        privkey = data.child(b"node.pem")
-        tub_maker = lambda handler_overrides: Tub(
-            certData=privkey.getContent(),
-        )
+        tub_maker = lambda handler_overrides: new_tub()
     return StorageFarmBroker(True, tub_maker, EMPTY_CLIENT_CONFIG)
 
 
@@ -593,7 +601,21 @@ storage:
 
     @inlineCallbacks
     def test_threshold_reached(self):
-        introducer = Mock()
+        """
+        ``StorageFarmBroker.when_connected_enough`` returns a ``Deferred`` which
+        only fires after the ``StorageFarmBroker`` has established at least as
+        many connections as requested.
+        """
+        introducer = MemoryIntroducerClient(
+            new_tub(),
+            SOME_FURL,
+            b"",
+            None,
+            None,
+            None,
+            None,
+            None,
+        )
         new_tubs = []
         def make_tub(*args, **kwargs):
             return new_tubs.pop()
@@ -601,10 +623,12 @@ storage:
         done = broker.when_connected_enough(5)
         broker.use_introducer(introducer)
         # subscribes to "storage" to learn of new storage nodes
-        subscribe = introducer.mock_calls[0]
-        self.assertEqual(subscribe[0], 'subscribe_to')
-        self.assertEqual(subscribe[1][0], 'storage')
-        got_announcement = subscribe[1][1]
+        [subscribe] = introducer.subscribed_to
+        self.assertEqual(
+            subscribe.service_name,
+            "storage",
+        )
+        got_announcement = subscribe.cb
 
         data = {
             "service-name": "storage",

From 80d2ecb815cb7b8f0ea24fff944c9fbbdcd03026 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Tue, 24 Nov 2020 14:52:17 -0500
Subject: [PATCH 43/67] news fragment

---
 newsfragments/3521.minor | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 create mode 100644 newsfragments/3521.minor

diff --git a/newsfragments/3521.minor b/newsfragments/3521.minor
new file mode 100644
index 000000000..e69de29bb

From 3843131acfc8a659a708368f3405ba2835bceb35 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Thu, 26 Nov 2020 19:29:52 -0500
Subject: [PATCH 44/67] Can have more than one introducer if you want

---
 docs/configuration.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/configuration.rst b/docs/configuration.rst
index 8583888ca..0aab9b395 100644
--- a/docs/configuration.rst
+++ b/docs/configuration.rst
@@ -399,8 +399,8 @@ This section controls *when* Tor and I2P are used. The ``[tor]`` and
 managed.
 
 All Tahoe nodes need to make a connection to the Introducer; the
-``private/introducers.yaml`` file (described below) configures where the
-Introducer lives. Tahoe client nodes must also make connections to storage
+``private/introducers.yaml`` file (described below) configures where one or more
+Introducers live. Tahoe client nodes must also make connections to storage
 servers: these targets are specified in announcements that come from the
 Introducer.  Both are expressed as FURLs (a Foolscap URL), which include a
 list of "connection hints". Each connection hint describes one (of perhaps

From a978fcf433816174c521a236361392c37b1f1f48 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Thu, 26 Nov 2020 19:35:39 -0500
Subject: [PATCH 45/67] Replace asserts with explicit checks and TypeError

---
 src/allmydata/node.py | 29 ++++++++++++++++++++++++++---
 1 file changed, 26 insertions(+), 3 deletions(-)

diff --git a/src/allmydata/node.py b/src/allmydata/node.py
index 7eefbbce0..6253aad4b 100644
--- a/src/allmydata/node.py
+++ b/src/allmydata/node.py
@@ -463,11 +463,34 @@ class _Config(object):
                     for petname, config
                     in introducers_yaml.get("introducers", {}).items()
                 }
-                assert all(isinstance(k, str) for k in introducers.keys())
-                assert all(isinstance(v, str) for v in introducers.values()), introducers.values()
+                non_strs = list(
+                    k
+                    for k
+                    in introducers.keys()
+                    if not isinstance(k, str)
+                )
+                if non_strs:
+                    raise TypeError(
+                        "Introducer petnames {!r} should have been str".format(
+                            non_strs,
+                        ),
+                    )
+                non_strs = list(
+                    v
+                    for v
+                    in introducers.values()
+                    if not isinstance(v, str)
+                )
+                if non_strs:
+                    raise TypeError(
+                        "Introducer fURLs {!r} should have been str".format(
+                            non_strs,
+                        ),
+                    )
                 log.msg(
-                    "found {} introducers in private/introducers.yaml".format(
+                    "found {} introducers in {!r}".format(
                         len(introducers),
+                        introducers_yaml_filename,
                     )
                 )
         except EnvironmentError as e:

From 805378ef1177f13c925a8a2dce2b626b2c8c16f6 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Thu, 26 Nov 2020 20:53:57 -0500
Subject: [PATCH 46/67] Do more path stuff with FilePath

---
 src/allmydata/scripts/common.py       |  2 +-
 src/allmydata/scripts/create_node.py  |  5 +-
 src/allmydata/test/check_memory.py    | 25 +++++----
 src/allmydata/test/common.py          |  2 +-
 src/allmydata/test/test_client.py     | 74 ++++++++++++++-------------
 src/allmydata/test/test_introducer.py | 35 +++++++------
 src/allmydata/test/test_system.py     | 15 +++---
 7 files changed, 86 insertions(+), 72 deletions(-)

diff --git a/src/allmydata/scripts/common.py b/src/allmydata/scripts/common.py
index 6d633b502..f38a241dd 100644
--- a/src/allmydata/scripts/common.py
+++ b/src/allmydata/scripts/common.py
@@ -124,7 +124,7 @@ def write_introducer(basedir, petname, furl):
     Overwrite the node's ``introducers.yaml`` with a file containing the given
     introducer information.
     """
-    FilePath(basedir).child(b"private").child(b"introducers.yaml").setContent(
+    basedir.child(b"private").child(b"introducers.yaml").setContent(
         safe_dump({
             "introducers": {
                 petname: {
diff --git a/src/allmydata/scripts/create_node.py b/src/allmydata/scripts/create_node.py
index 30024dfd9..a4b2213ed 100644
--- a/src/allmydata/scripts/create_node.py
+++ b/src/allmydata/scripts/create_node.py
@@ -5,6 +5,9 @@ import json
 
 from twisted.internet import reactor, defer
 from twisted.python.usage import UsageError
+from twisted.python.filepath import (
+    FilePath,
+)
 
 from allmydata.scripts.common import (
     BasedirOptions,
@@ -308,7 +311,7 @@ def write_client_config(c, config):
     introducer = config.get("introducer", None)
     if introducer is not None:
         write_introducer(
-            config["basedir"],
+            FilePath(config["basedir"]),
             "default",
             introducer,
         )
diff --git a/src/allmydata/test/check_memory.py b/src/allmydata/test/check_memory.py
index 3ab88b9c6..6ec90eeae 100644
--- a/src/allmydata/test/check_memory.py
+++ b/src/allmydata/test/check_memory.py
@@ -8,6 +8,9 @@ if PY2:
     from future.builtins import str  # noqa: F401
 from six.moves import cStringIO as StringIO
 
+from twisted.python.filepath import (
+    FilePath,
+)
 from twisted.internet import defer, reactor, protocol, error
 from twisted.application import service, internet
 from twisted.web import client as tw_client
@@ -184,15 +187,17 @@ class SystemFramework(pollmixin.PollMixin):
         self.introducer_furl = self.introducer.introducer_url
 
     def make_nodes(self):
+        root = FilePath(self.testdir)
         self.nodes = []
         for i in range(self.numnodes):
-            nodedir = os.path.join(self.testdir, "node%d" % i)
-            os.makedirs(nodedir + b"/private")
+            nodedir = root.child("node%d" % (i,))
+            private = nodedir.child("private")
+            private.makedirs()
             write_introducer(nodedir, "default", self.introducer_url)
-            f = open(os.path.join(nodedir, "tahoe.cfg"), "w")
-            f.write("[client]\n"
-                    "shares.happy = 1\n"
-                    "[storage]\n"
+            config = (
+                "[client]\n"
+                "shares.happy = 1\n"
+                "[storage]\n"
             )
             # the only tests for which we want the internal nodes to actually
             # retain shares are the ones where somebody's going to download
@@ -204,13 +209,13 @@ class SystemFramework(pollmixin.PollMixin):
                 # for these tests, we tell the storage servers to pretend to
                 # accept shares, but really just throw them out, since we're
                 # only testing upload and not download.
-                f.write("debug_discard = true\n")
+                config += "debug_discard = true\n"
             if self.mode in ("receive",):
                 # for this mode, the client-under-test gets all the shares,
                 # so our internal nodes can refuse requests
-                f.write("readonly = true\n")
-            f.close()
-            c = client.Client(basedir=nodedir)
+                config += "readonly = true\n"
+            nodedir.child("tahoe.cfg").setContent(config)
+            c = client.Client(basedir=nodedir.path)
             c.setServiceParent(self)
             self.nodes.append(c)
         # the peers will start running, eventually they will connect to each
diff --git a/src/allmydata/test/common.py b/src/allmydata/test/common.py
index cbe4d768c..9691877fd 100644
--- a/src/allmydata/test/common.py
+++ b/src/allmydata/test/common.py
@@ -251,7 +251,7 @@ class UseNode(object):
 )
 
         write_introducer(
-            self.basedir.asBytesMode().path,
+            self.basedir,
             "default",
             self.introducer_furl,
         )
diff --git a/src/allmydata/test/test_client.py b/src/allmydata/test/test_client.py
index e082a33d1..080716507 100644
--- a/src/allmydata/test/test_client.py
+++ b/src/allmydata/test/test_client.py
@@ -667,11 +667,11 @@ class AnonymousStorage(SyncTestCase):
         """
         If anonymous storage access is enabled then the client announces it.
         """
-        basedir = self.id()
-        os.makedirs(basedir + b"/private")
+        basedir = FilePath(self.id())
+        basedir.child("private").makedirs()
         write_introducer(basedir, "someintroducer", SOME_FURL)
         config = client.config_from_string(
-            basedir,
+            basedir.path,
             "tub.port",
             BASECONFIG + (
                 "[storage]\n"
@@ -687,7 +687,7 @@ class AnonymousStorage(SyncTestCase):
             get_published_announcements(node),
             MatchesListwise([
                 matches_storage_announcement(
-                    basedir,
+                    basedir.path,
                     anonymous=True,
                 ),
             ]),
@@ -699,11 +699,11 @@ class AnonymousStorage(SyncTestCase):
         If anonymous storage access is disabled then the client does not announce
         it nor does it write a fURL for it to beneath the node directory.
         """
-        basedir = self.id()
-        os.makedirs(basedir + b"/private")
+        basedir = FilePath(self.id())
+        basedir.child("private").makedirs()
         write_introducer(basedir, "someintroducer", SOME_FURL)
         config = client.config_from_string(
-            basedir,
+            basedir.path,
             "tub.port",
             BASECONFIG + (
                 "[storage]\n"
@@ -719,7 +719,7 @@ class AnonymousStorage(SyncTestCase):
             get_published_announcements(node),
             MatchesListwise([
                 matches_storage_announcement(
-                    basedir,
+                    basedir.path,
                     anonymous=False,
                 ),
             ]),
@@ -737,10 +737,10 @@ class AnonymousStorage(SyncTestCase):
         possible to reach the anonymous storage server via the originally
         published fURL.
         """
-        basedir = self.id()
-        os.makedirs(basedir + b"/private")
+        basedir = FilePath(self.id())
+        basedir.child("private").makedirs()
         enabled_config = client.config_from_string(
-            basedir,
+            basedir.path,
             "tub.port",
             BASECONFIG + (
                 "[storage]\n"
@@ -764,7 +764,7 @@ class AnonymousStorage(SyncTestCase):
         )
 
         disabled_config = client.config_from_string(
-            basedir,
+            basedir.path,
             "tub.port",
             BASECONFIG + (
                 "[storage]\n"
@@ -956,22 +956,24 @@ class Run(unittest.TestCase, testutil.StallMixin):
         A configuration consisting only of an introducer can be turned into a
         client node.
         """
-        basedir = "test_client.Run.test_loadable"
-        os.makedirs(basedir + b"/private")
+        basedir = FilePath("test_client.Run.test_loadable")
+        private = basedir.child("private")
+        private.makedirs()
         dummy = "pb://wl74cyahejagspqgy4x5ukrvfnevlknt@127.0.0.1:58889/bogus"
         write_introducer(basedir, "someintroducer", dummy)
-        fileutil.write(os.path.join(basedir, "tahoe.cfg"), BASECONFIG)
-        fileutil.write(os.path.join(basedir, client._Client.EXIT_TRIGGER_FILE), "")
-        yield client.create_client(basedir)
+        basedir.child("tahoe.cfg").setContent(BASECONFIG)
+        basedir.child(client._Client.EXIT_TRIGGER_FILE).touch()
+        yield client.create_client(basedir.path)
 
     @defer.inlineCallbacks
     def test_reloadable(self):
-        basedir = "test_client.Run.test_reloadable"
-        os.makedirs(basedir + b"/private")
+        basedir = FilePath("test_client.Run.test_reloadable")
+        private = basedir.child("private")
+        private.makedirs()
         dummy = "pb://wl74cyahejagspqgy4x5ukrvfnevlknt@127.0.0.1:58889/bogus"
         write_introducer(basedir, "someintroducer", dummy)
-        fileutil.write(os.path.join(basedir, "tahoe.cfg"), BASECONFIG)
-        c1 = yield client.create_client(basedir)
+        basedir.child("tahoe.cfg").setContent(BASECONFIG)
+        c1 = yield client.create_client(basedir.path)
         c1.setServiceParent(self.sparent)
 
         # delay to let the service start up completely. I'm not entirely sure
@@ -993,7 +995,7 @@ class Run(unittest.TestCase, testutil.StallMixin):
         # also change _check_exit_trigger to use it instead of a raw
         # reactor.stop, also instrument the shutdown event in an
         # attribute that we can check.)
-        c2 = yield client.create_client(basedir)
+        c2 = yield client.create_client(basedir.path)
         c2.setServiceParent(self.sparent)
         yield c2.disownServiceParent()
 
@@ -1132,8 +1134,8 @@ class StorageAnnouncementTests(SyncTestCase):
     """
     def setUp(self):
         super(StorageAnnouncementTests, self).setUp()
-        self.basedir = self.useFixture(TempDir()).path
-        create_node_dir(self.basedir, u"")
+        self.basedir = FilePath(self.useFixture(TempDir()).path)
+        create_node_dir(self.basedir.path, u"")
         # Write an introducer configuration or we can't observer
         # announcements.
         write_introducer(self.basedir, "someintroducer", SOME_FURL)
@@ -1164,7 +1166,7 @@ enabled = {storage_enabled}
         No storage announcement is published if storage is not enabled.
         """
         config = client.config_from_string(
-            self.basedir,
+            self.basedir.path,
             "tub.port",
             self.get_config(storage_enabled=False),
         )
@@ -1186,7 +1188,7 @@ enabled = {storage_enabled}
         storage is enabled.
         """
         config = client.config_from_string(
-            self.basedir,
+            self.basedir.path,
             "tub.port",
             self.get_config(storage_enabled=True),
         )
@@ -1203,7 +1205,7 @@ enabled = {storage_enabled}
                 # Match the following list (of one element) ...
                 MatchesListwise([
                     # The only element in the list ...
-                    matches_storage_announcement(self.basedir),
+                    matches_storage_announcement(self.basedir.path),
                 ]),
             )),
         )
@@ -1218,7 +1220,7 @@ enabled = {storage_enabled}
 
         value = u"thing"
         config = client.config_from_string(
-            self.basedir,
+            self.basedir.path,
             "tub.port",
             self.get_config(
                 storage_enabled=True,
@@ -1238,7 +1240,7 @@ enabled = {storage_enabled}
                 get_published_announcements,
                 MatchesListwise([
                     matches_storage_announcement(
-                        self.basedir,
+                        self.basedir.path,
                         options=[
                             matches_dummy_announcement(
                                 u"tahoe-lafs-dummy-v1",
@@ -1259,7 +1261,7 @@ enabled = {storage_enabled}
         self.useFixture(UseTestPlugins())
 
         config = client.config_from_string(
-            self.basedir,
+            self.basedir.path,
             "tub.port",
             self.get_config(
                 storage_enabled=True,
@@ -1281,7 +1283,7 @@ enabled = {storage_enabled}
                 get_published_announcements,
                 MatchesListwise([
                     matches_storage_announcement(
-                        self.basedir,
+                        self.basedir.path,
                         options=[
                             matches_dummy_announcement(
                                 u"tahoe-lafs-dummy-v1",
@@ -1307,7 +1309,7 @@ enabled = {storage_enabled}
         self.useFixture(UseTestPlugins())
 
         config = client.config_from_string(
-            self.basedir,
+            self.basedir.path,
             "tub.port",
             self.get_config(
                 storage_enabled=True,
@@ -1343,7 +1345,7 @@ enabled = {storage_enabled}
         self.useFixture(UseTestPlugins())
 
         config = client.config_from_string(
-            self.basedir,
+            self.basedir.path,
             "tub.port",
             self.get_config(
                 storage_enabled=True,
@@ -1359,7 +1361,7 @@ enabled = {storage_enabled}
                 get_published_announcements,
                 MatchesListwise([
                     matches_storage_announcement(
-                        self.basedir,
+                        self.basedir.path,
                         options=[
                             matches_dummy_announcement(
                                 u"tahoe-lafs-dummy-v1",
@@ -1381,7 +1383,7 @@ enabled = {storage_enabled}
         self.useFixture(UseTestPlugins())
 
         config = client.config_from_string(
-            self.basedir,
+            self.basedir.path,
             "tub.port",
             self.get_config(
                 storage_enabled=True,
@@ -1408,7 +1410,7 @@ enabled = {storage_enabled}
         available on the system.
         """
         config = client.config_from_string(
-            self.basedir,
+            self.basedir.path,
             "tub.port",
             self.get_config(
                 storage_enabled=True,
diff --git a/src/allmydata/test/test_introducer.py b/src/allmydata/test/test_introducer.py
index f6497eeb4..1fb456e72 100644
--- a/src/allmydata/test/test_introducer.py
+++ b/src/allmydata/test/test_introducer.py
@@ -795,21 +795,24 @@ class Announcements(AsyncTestCase):
         Announcements received by an introducer client are written to that
         introducer client's cache file.
         """
-        basedir = "introducer/ClientSeqnums/test_client_cache_1"
-        fileutil.make_dirs(basedir + b"/private")
+        basedir = FilePath("introducer/ClientSeqnums/test_client_cache_1")
+        private = basedir.child("private")
+        private.makedirs()
         write_introducer(basedir, "default", "nope")
-        cache_filepath = FilePath(os.path.join(basedir, "private",
-                                               "introducer_default_cache.yaml"))
+        cache_filepath = basedir.descendant([
+            "private",
+            "introducer_default_cache.yaml",
+        ])
 
         # if storage is enabled, the Client will publish its storage server
         # during startup (although the announcement will wait in a queue
         # until the introducer connection is established). To avoid getting
         # confused by this, disable storage.
-        with open(os.path.join(basedir, "tahoe.cfg"), "w") as f:
+        with basedir.child("tahoe.cfg").open("w") as f:
             f.write("[storage]\n")
             f.write("enabled = false\n")
 
-        c = yield create_client(basedir)
+        c = yield create_client(basedir.path)
         ic = c.introducer_clients[0]
         private_key, public_key = ed25519.create_signing_keypair()
         public_key_str = remove_prefix(ed25519.string_from_verifying_key(public_key), "pub-")
@@ -875,7 +878,7 @@ class Announcements(AsyncTestCase):
         self.failUnlessEqual(announcements[public_key_str2]["anonymous-storage-FURL"],
                              furl3)
 
-        c2 = yield create_client(basedir)
+        c2 = yield create_client(basedir.path)
         c2.introducer_clients[0]._load_announcements()
         yield flushEventualQueue()
         self.assertEqual(c2.storage_broker.get_all_serverids(),
@@ -885,26 +888,24 @@ class ClientSeqnums(AsyncBrokenTestCase):
 
     @defer.inlineCallbacks
     def test_client(self):
-        basedir = "introducer/ClientSeqnums/test_client"
-        fileutil.make_dirs(basedir + b"/private")
+        basedir = FilePath("introducer/ClientSeqnums/test_client")
+        private = basedir.child("private")
+        private.makedirs()
         write_introducer(basedir, "default", "nope")
         # if storage is enabled, the Client will publish its storage server
         # during startup (although the announcement will wait in a queue
         # until the introducer connection is established). To avoid getting
         # confused by this, disable storage.
-        f = open(os.path.join(basedir, "tahoe.cfg"), "w")
-        f.write("[storage]\n")
-        f.write("enabled = false\n")
-        f.close()
+        with basedir.child("tahoe.cfg").open("w") as f:
+            f.write("[storage]\n")
+            f.write("enabled = false\n")
 
-        c = yield create_client(basedir)
+        c = yield create_client(basedir.path)
         ic = c.introducer_clients[0]
         outbound = ic._outbound_announcements
         published = ic._published_announcements
         def read_seqnum():
-            f = open(os.path.join(basedir, "announcement-seqnum"))
-            seqnum = f.read().strip()
-            f.close()
+            seqnum = basedir.child("announcement-seqnum").getContent()
             return int(seqnum)
 
         ic.publish("sA", {"key": "value1"}, c._node_private_key)
diff --git a/src/allmydata/test/test_system.py b/src/allmydata/test/test_system.py
index 5a34dcc4d..7a7fe117b 100644
--- a/src/allmydata/test/test_system.py
+++ b/src/allmydata/test/test_system.py
@@ -33,6 +33,9 @@ from allmydata.mutable.publish import MutableData
 
 from foolscap.api import DeadReferenceError, fireEventually, flushEventualQueue
 from twisted.python.failure import Failure
+from twisted.python.filepath import (
+    FilePath,
+)
 
 from .common import (
     TEST_RSA_KEY_SIZE,
@@ -903,21 +906,21 @@ class SystemTestMixin(pollmixin.PollMixin, testutil.StallMixin):
         # usually this node is *not* parented to our self.sparent, so we can
         # shut it down separately from the rest, to exercise the
         # connection-lost code
-        basedir = self.getdir("client%d" % client_num)
-        if not os.path.isdir(basedir):
-            fileutil.make_dirs(basedir)
+        basedir = FilePath(self.getdir("client%d" % client_num))
+        basedir.makedirs()
         config = "[client]\n"
         if helper_furl:
             config += "helper.furl = %s\n" % helper_furl
-        fileutil.write(os.path.join(basedir, 'tahoe.cfg'), config)
-        os.makedirs(basedir + b"/private")
+        basedir.child("tahoe.cfg").setContent(config)
+        private = basedir.child("private")
+        private.makedirs()
         write_introducer(
             basedir,
             "default",
             self.introducer_furl,
         )
 
-        c = yield client.create_client(basedir)
+        c = yield client.create_client(basedir.path)
         self.clients.append(c)
         c.set_default_mutable_keysize(TEST_RSA_KEY_SIZE)
         self.numclients += 1

From 84088e4f4182f0fdb52b995f7db8f7d8b90ea06b Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Thu, 26 Nov 2020 21:18:58 -0500
Subject: [PATCH 47/67] unused import

---
 src/allmydata/scripts/common.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/src/allmydata/scripts/common.py b/src/allmydata/scripts/common.py
index f38a241dd..b20cca65f 100644
--- a/src/allmydata/scripts/common.py
+++ b/src/allmydata/scripts/common.py
@@ -14,9 +14,6 @@ if PY2:
     from future.builtins import str  # noqa: F401
 
 from twisted.python import usage
-from twisted.python.filepath import FilePath
-
-
 
 from allmydata.util.assertutil import precondition
 from allmydata.util.encodingutil import unicode_to_url, quote_output, \

From ae5351c2040724b2e523b2706b65000c9653d284 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Thu, 26 Nov 2020 21:46:57 -0500
Subject: [PATCH 48/67] Adapt test_tor to write_introducer change

---
 integration/test_tor.py | 16 ++++++++++------
 1 file changed, 10 insertions(+), 6 deletions(-)

diff --git a/integration/test_tor.py b/integration/test_tor.py
index 8dd05fc3f..af657f0ad 100644
--- a/integration/test_tor.py
+++ b/integration/test_tor.py
@@ -9,6 +9,10 @@ import pytest_twisted
 
 import util
 
+from twisted.python.filepath import (
+    FilePath,
+)
+
 from allmydata.test.common import (
     write_introducer,
 )
@@ -70,12 +74,12 @@ def test_onion_service_storage(reactor, request, temp_dir, flog_gatherer, tor_ne
 
 @pytest_twisted.inlineCallbacks
 def _create_anonymous_node(reactor, name, control_port, request, temp_dir, flog_gatherer, tor_network, introducer_furl):
-    node_dir = join(temp_dir, name)
+    node_dir = FilePath(temp_dir).child(name)
     web_port = "tcp:{}:interface=localhost".format(control_port + 2000)
 
     if True:
-        print("creating", node_dir)
-        mkdir(node_dir)
+        print("creating", node_dir.path)
+        node_dir.makedirs()
         proto = util._DumpOutputProtocol(None)
         reactor.spawnProcess(
             proto,
@@ -88,7 +92,7 @@ def _create_anonymous_node(reactor, name, control_port, request, temp_dir, flog_
                 '--hide-ip',
                 '--tor-control-port', 'tcp:localhost:{}'.format(control_port),
                 '--listen', 'tor',
-                node_dir,
+                node_dir.path,
             )
         )
         yield proto.done
@@ -96,7 +100,7 @@ def _create_anonymous_node(reactor, name, control_port, request, temp_dir, flog_
 
     # Which services should this client connect to?
     write_introducer(node_dir, "default", introducer_furl)
-    with open(join(node_dir, 'tahoe.cfg'), 'w') as f:
+    with node_dir.child('tahoe.cfg').open('w') as f:
         f.write('''
 [node]
 nickname = %(name)s
@@ -125,5 +129,5 @@ shares.total = 2
 })
 
     print("running")
-    yield util._run_node(reactor, node_dir, request, None)
+    yield util._run_node(reactor, node_dir.path, request, None)
     print("okay, launched")

From 4c8fb8d93aa5715856c47bc1d57155b4117b87ad Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Thu, 26 Nov 2020 21:48:06 -0500
Subject: [PATCH 49/67] unused import

---
 integration/test_tor.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/integration/test_tor.py b/integration/test_tor.py
index af657f0ad..dcbfb1151 100644
--- a/integration/test_tor.py
+++ b/integration/test_tor.py
@@ -1,7 +1,6 @@
 from __future__ import print_function
 
 import sys
-from os import mkdir
 from os.path import join
 
 import pytest

From 263ada9be4b0c8e7f98109ab679251733ad88c9f Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Fri, 27 Nov 2020 16:24:16 -0500
Subject: [PATCH 50/67] Get rid of the spurious quotes in the flake8 command

I don't understand tox.ini syntax or quoting rules and I don't see any
documentation about it.  But what could go wrong with trial and error?
---
 tox.ini | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tox.ini b/tox.ini
index b95476f58..5b9a146fe 100644
--- a/tox.ini
+++ b/tox.ini
@@ -98,9 +98,9 @@ setenv =
 
 	 # If no positional arguments are given, try to run the checks on the
 	 # entire codebase, including various pieces of supporting code.
-	 DEFAULT_FILES="src integration static misc setup.py"
+	 DEFAULT_FILES=src integration static misc setup.py
 commands =
-         flake8 {posargs:{env:DEFAULT_FILES}}
+         flake8 -v {posargs:{env:DEFAULT_FILES}}
          python misc/coding_tools/check-umids.py {posargs:{env:DEFAULT_FILES}}
          python misc/coding_tools/check-debugging.py {posargs:{env:DEFAULT_FILES}}
          python misc/coding_tools/find-trailing-spaces.py -r {posargs:{env:DEFAULT_FILES}}

From e6a09fa4441a3d354cc0d806516e472b64917489 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Fri, 27 Nov 2020 16:28:23 -0500
Subject: [PATCH 51/67] Don't check check-debugging.py

---
 misc/coding_tools/check-debugging.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/misc/coding_tools/check-debugging.py b/misc/coding_tools/check-debugging.py
index 17eeb30b7..f2ba6528e 100755
--- a/misc/coding_tools/check-debugging.py
+++ b/misc/coding_tools/check-debugging.py
@@ -11,8 +11,12 @@ umids = {}
 
 for starting_point in sys.argv[1:]:
     for root, dirs, files in os.walk(starting_point):
-        for fn in [f for f in files if f.endswith(".py")]:
-            fn = os.path.join(root, fn)
+        for f in files:
+            if not f.endswith(".py"):
+                continue
+            if f == "check-debugging.py":
+                continue
+            fn = os.path.join(root, f)
             for lineno,line in enumerate(open(fn, "r").readlines()):
                 lineno = lineno+1
                 mo = re.search(r"\.setDebugging\(True\)", line)

From b02a4f73b6c5fdfdd3f81b0bf85c92b58e2b6fb7 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Mon, 30 Nov 2020 08:56:37 -0500
Subject: [PATCH 52/67] news fragment

---
 newsfragments/3539.bugfix | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 newsfragments/3539.bugfix

diff --git a/newsfragments/3539.bugfix b/newsfragments/3539.bugfix
new file mode 100644
index 000000000..ed4aeb9af
--- /dev/null
+++ b/newsfragments/3539.bugfix
@@ -0,0 +1 @@
+Certain implementation-internal weakref KeyErrors are now handled and should no longer cause user-initiated operations to fail.

From 01ab8d3ee94119855760c384b62c7bf7f41c5aa2 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Mon, 30 Nov 2020 08:56:45 -0500
Subject: [PATCH 53/67] Don't look before you leap

---
 src/allmydata/nodemaker.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/allmydata/nodemaker.py b/src/allmydata/nodemaker.py
index 8e68d92fe..c3ba1ba7b 100644
--- a/src/allmydata/nodemaker.py
+++ b/src/allmydata/nodemaker.py
@@ -66,9 +66,9 @@ class NodeMaker(object):
             memokey = b"I" + bigcap
         else:
             memokey = b"M" + bigcap
-        if memokey in self._node_cache:
+        try:
             node = self._node_cache[memokey]
-        else:
+        except KeyError:
             cap = uri.from_string(bigcap, deep_immutable=deep_immutable,
                                   name=name)
             node = self._create_from_single_cap(cap)

From 4ca45aaa933f8a79ddf11fa50a91080c411c7402 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Mon, 30 Nov 2020 13:23:18 -0500
Subject: [PATCH 54/67] Catch basedir type errors earlier

---
 src/allmydata/test/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/allmydata/test/common.py b/src/allmydata/test/common.py
index a420dd3ba..34fca6481 100644
--- a/src/allmydata/test/common.py
+++ b/src/allmydata/test/common.py
@@ -221,7 +221,7 @@ class UseNode(object):
     """
     plugin_config = attr.ib()
     storage_plugin = attr.ib()
-    basedir = attr.ib()
+    basedir = attr.ib(validator=attr.validators.instance_of(FilePath))
     introducer_furl = attr.ib()
     node_config = attr.ib(default=attr.Factory(dict))
 

From 2ac4af7fb418f15fa86c33c235bcfb5560aec657 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Mon, 30 Nov 2020 13:26:32 -0500
Subject: [PATCH 55/67] Add some direct tests for `NodeMaker.create_from_uri`

---
 src/allmydata/test/strategies.py  | 111 ++++++++++++++++++++++++++++++
 src/allmydata/test/test_client.py | 110 ++++++++++++++++++++++++++++-
 2 files changed, 220 insertions(+), 1 deletion(-)
 create mode 100644 src/allmydata/test/strategies.py

diff --git a/src/allmydata/test/strategies.py b/src/allmydata/test/strategies.py
new file mode 100644
index 000000000..553b2c226
--- /dev/null
+++ b/src/allmydata/test/strategies.py
@@ -0,0 +1,111 @@
+"""
+Hypothesis strategies use for testing Tahoe-LAFS.
+"""
+
+from hypothesis.strategies import (
+    one_of,
+    builds,
+    binary,
+)
+
+from ..uri import (
+    WriteableSSKFileURI,
+    WriteableMDMFFileURI,
+    DirectoryURI,
+    MDMFDirectoryURI,
+)
+
+def write_capabilities():
+    """
+    Build ``IURI`` providers representing all kinds of write capabilities.
+    """
+    return one_of([
+        ssk_capabilities(),
+        mdmf_capabilities(),
+        dir2_capabilities(),
+        dir2_mdmf_capabilities(),
+    ])
+
+
+def ssk_capabilities():
+    """
+    Build ``WriteableSSKFileURI`` instances.
+    """
+    return builds(
+        WriteableSSKFileURI,
+        ssk_writekeys(),
+        ssk_fingerprints(),
+    )
+
+
+def _writekeys(size=16):
+    """
+    Build ``bytes`` representing write keys.
+    """
+    return binary(min_size=size, max_size=size)
+
+
+def ssk_writekeys():
+    """
+    Build ``bytes`` representing SSK write keys.
+    """
+    return _writekeys()
+
+
+def _fingerprints(size=32):
+    """
+    Build ``bytes`` representing fingerprints.
+    """
+    return binary(min_size=size, max_size=size)
+
+
+def ssk_fingerprints():
+    """
+    Build ``bytes`` representing SSK fingerprints.
+    """
+    return _fingerprints()
+
+
+def mdmf_capabilities():
+    """
+    Build ``WriteableMDMFFileURI`` instances.
+    """
+    return builds(
+        WriteableMDMFFileURI,
+        mdmf_writekeys(),
+        mdmf_fingerprints(),
+    )
+
+
+def mdmf_writekeys():
+    """
+    Build ``bytes`` representing MDMF write keys.
+    """
+    return _writekeys()
+
+
+def mdmf_fingerprints():
+    """
+    Build ``bytes`` representing MDMF fingerprints.
+    """
+    return _fingerprints()
+
+
+def dir2_capabilities():
+    """
+    Build ``DirectoryURI`` instances.
+    """
+    return builds(
+        DirectoryURI,
+        ssk_capabilities(),
+    )
+
+
+def dir2_mdmf_capabilities():
+    """
+    Build ``MDMFDirectoryURI`` instances.
+    """
+    return builds(
+        MDMFDirectoryURI,
+        mdmf_capabilities(),
+    )
diff --git a/src/allmydata/test/test_client.py b/src/allmydata/test/test_client.py
index 54c5be8e5..1e42c54aa 100644
--- a/src/allmydata/test/test_client.py
+++ b/src/allmydata/test/test_client.py
@@ -12,6 +12,15 @@ from fixtures import (
     Fixture,
     TempDir,
 )
+
+from hypothesis import (
+    given,
+)
+from hypothesis.strategies import (
+    sampled_from,
+    booleans,
+)
+
 from eliot.testing import (
     capture_logging,
     assertHasAction,
@@ -39,6 +48,9 @@ from testtools.twistedsupport import (
 import allmydata
 import allmydata.util.log
 
+from allmydata.nodemaker import (
+    NodeMaker,
+)
 from allmydata.node import OldConfigError, UnescapedHashError, create_node_dir
 from allmydata.frontends.auth import NeedRootcapLookupScheme
 from allmydata import client
@@ -60,7 +72,9 @@ import allmydata.test.common_util as testutil
 from .common import (
     EMPTY_CLIENT_CONFIG,
     SyncTestCase,
+    AsyncBrokenTestCase,
     UseTestPlugins,
+    UseNode,
     MemoryIntroducerClient,
     get_published_announcements,
 )
@@ -69,6 +83,9 @@ from .matchers import (
     matches_storage_announcement,
     matches_furl,
 )
+from .strategies import (
+    write_capabilities,
+)
 
 SOME_FURL = b"pb://abcde@nowhere/fake"
 
@@ -987,7 +1004,98 @@ class Run(unittest.TestCase, testutil.StallMixin):
         c2.setServiceParent(self.sparent)
         yield c2.disownServiceParent()
 
-class NodeMaker(testutil.ReallyEqualMixin, unittest.TestCase):
+class NodeMakerTests(testutil.ReallyEqualMixin, AsyncBrokenTestCase):
+
+    def _make_node_maker(self, mode, writecap, deep_immutable):
+        """
+        Create a callable which can create an ``IFilesystemNode`` provider for the
+        given cap.
+
+        :param unicode mode: The read/write combination to pass to
+            ``NodeMaker.create_from_cap``.  If it contains ``u"r"`` then a
+            readcap will be passed in.  If it contains ``u"w"`` then a
+            writecap will be passed in.
+
+        :param IURI writecap: The capability for which to create a node.
+
+        :param bool deep_immutable: Whether to request a "deep immutable" node
+            which forces the result to be an immutable ``IFilesystemNode`` (I
+            think -exarkun).
+        """
+        if writecap.is_mutable():
+            # It's just not a valid combination to have a mutable alongside
+            # deep_immutable = True.  It's easier to fix deep_immutable than
+            # writecap to clear up this conflict.
+            deep_immutable = False
+
+        if "r" in mode:
+            readcap = writecap.get_readonly().to_string()
+        else:
+            readcap = None
+        if "w" in mode:
+            writecap = writecap.to_string()
+        else:
+            writecap = None
+
+        nm = NodeMaker(
+            storage_broker=None,
+            secret_holder=None,
+            history=None,
+            uploader=None,
+            terminator=None,
+            default_encoding_parameters={u"k": 1, u"n": 1},
+            mutable_file_default=None,
+            key_generator=None,
+            blacklist=None,
+        )
+        return partial(
+            nm.create_from_cap,
+            writecap,
+            readcap,
+            deep_immutable,
+        )
+
+    @given(
+        mode=sampled_from(["w", "r", "rw"]),
+        writecap=write_capabilities(),
+        deep_immutable=booleans(),
+    )
+    def test_cached_result(self, mode, writecap, deep_immutable):
+        """
+        ``NodeMaker.create_from_cap`` returns the same object when called with the
+        same arguments.
+        """
+        make_node = self._make_node_maker(mode, writecap, deep_immutable)
+        original = make_node()
+        additional = make_node()
+
+        self.assertThat(
+            original,
+            Is(additional),
+        )
+
+    @given(
+        mode=sampled_from(["w", "r", "rw"]),
+        writecap=write_capabilities(),
+        deep_immutable=booleans(),
+    )
+    def test_cache_expired(self, mode, writecap, deep_immutable):
+        """
+        After the node object returned by an earlier call to
+        ``NodeMaker.create_from_cap`` has been garbage collected, a new call
+        to ``NodeMaker.create_from_cap`` returns a node object, maybe even a
+        new one although we can't really prove it.
+        """
+        make_node = self._make_node_maker(mode, writecap, deep_immutable)
+        make_node()
+        additional = make_node()
+        self.assertThat(
+            additional,
+            AfterPreprocessing(
+                lambda node: node.get_readonly_uri(),
+                Equals(writecap.get_readonly().to_string()),
+            ),
+        )
 
     @defer.inlineCallbacks
     def test_maker(self):

From ef2f7e61364c6a3187d2ab4859adfc4031213bdd Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Mon, 30 Nov 2020 13:27:46 -0500
Subject: [PATCH 56/67] unused import

---
 src/allmydata/test/test_client.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/allmydata/test/test_client.py b/src/allmydata/test/test_client.py
index 1e42c54aa..98555783f 100644
--- a/src/allmydata/test/test_client.py
+++ b/src/allmydata/test/test_client.py
@@ -74,7 +74,6 @@ from .common import (
     SyncTestCase,
     AsyncBrokenTestCase,
     UseTestPlugins,
-    UseNode,
     MemoryIntroducerClient,
     get_published_announcements,
 )

From 17ed8afd2f7a33bb38a2d2249f1754662898f673 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Mon, 30 Nov 2020 13:37:21 -0500
Subject: [PATCH 57/67] Make the new test data files installable

There's an existing rule that matches *.txt
---
 .../data/{openssh-rsa-2048.pub => openssh-rsa-2048.pub.txt}   | 0
 .../test/data/{openssh-rsa-2048 => openssh-rsa-2048.txt}      | 0
 src/allmydata/test/test_client.py                             | 4 ++--
 3 files changed, 2 insertions(+), 2 deletions(-)
 rename src/allmydata/test/data/{openssh-rsa-2048.pub => openssh-rsa-2048.pub.txt} (100%)
 rename src/allmydata/test/data/{openssh-rsa-2048 => openssh-rsa-2048.txt} (100%)

diff --git a/src/allmydata/test/data/openssh-rsa-2048.pub b/src/allmydata/test/data/openssh-rsa-2048.pub.txt
similarity index 100%
rename from src/allmydata/test/data/openssh-rsa-2048.pub
rename to src/allmydata/test/data/openssh-rsa-2048.pub.txt
diff --git a/src/allmydata/test/data/openssh-rsa-2048 b/src/allmydata/test/data/openssh-rsa-2048.txt
similarity index 100%
rename from src/allmydata/test/data/openssh-rsa-2048
rename to src/allmydata/test/data/openssh-rsa-2048.txt
diff --git a/src/allmydata/test/test_client.py b/src/allmydata/test/test_client.py
index f6d3d4b84..3fdec8a5f 100644
--- a/src/allmydata/test/test_client.py
+++ b/src/allmydata/test/test_client.py
@@ -427,8 +427,8 @@ class Basic(testutil.ReallyEqualMixin, unittest.TestCase):
         accounts.touch()
 
         data = FilePath(__file__).sibling(b"data")
-        privkey = data.child(b"openssh-rsa-2048")
-        pubkey = data.child(b"openssh-rsa-2048.pub")
+        privkey = data.child(b"openssh-rsa-2048.txt")
+        pubkey = data.child(b"openssh-rsa-2048.pub.txt")
 
         basedir = u"client.Basic.test_ftp_create"
         create_node_dir(basedir, "testing")

From d50a1151bc5fdf73de4e25c880cd42ccdc1a34b3 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Tue, 1 Dec 2020 09:45:02 -0500
Subject: [PATCH 58/67] verbose is kind of annoying for normal use

---
 tox.ini | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tox.ini b/tox.ini
index 5b9a146fe..c61331885 100644
--- a/tox.ini
+++ b/tox.ini
@@ -100,7 +100,7 @@ setenv =
 	 # entire codebase, including various pieces of supporting code.
 	 DEFAULT_FILES=src integration static misc setup.py
 commands =
-         flake8 -v {posargs:{env:DEFAULT_FILES}}
+         flake8 {posargs:{env:DEFAULT_FILES}}
          python misc/coding_tools/check-umids.py {posargs:{env:DEFAULT_FILES}}
          python misc/coding_tools/check-debugging.py {posargs:{env:DEFAULT_FILES}}
          python misc/coding_tools/find-trailing-spaces.py -r {posargs:{env:DEFAULT_FILES}}

From 272d6d0aef3989018d41aaad4085824d311143e0 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Tue, 1 Dec 2020 09:52:38 -0500
Subject: [PATCH 59/67] Update developer docs wrt pre-commit

---
 Makefile                 | 16 +---------
 docs/developer-guide.rst | 67 ++++++----------------------------------
 2 files changed, 11 insertions(+), 72 deletions(-)

diff --git a/Makefile b/Makefile
index b48e74b0e..f7a357588 100644
--- a/Makefile
+++ b/Makefile
@@ -13,8 +13,6 @@ MAKEFLAGS += --warn-undefined-variables
 MAKEFLAGS += --no-builtin-rules
 
 # Local target variables
-VCS_HOOK_SAMPLES=$(wildcard .git/hooks/*.sample)
-VCS_HOOKS=$(VCS_HOOK_SAMPLES:%.sample=%)
 PYTHON=python
 export PYTHON
 PYFLAKES=flake8
@@ -31,15 +29,6 @@ TEST_SUITE=allmydata
 default:
 	@echo "no default target"
 
-.PHONY: install-vcs-hooks
-## Install the VCS hooks to run linters on commit and all tests on push
-install-vcs-hooks: .git/hooks/pre-commit .git/hooks/pre-push
-.PHONY: uninstall-vcs-hooks
-## Remove the VCS hooks
-uninstall-vcs-hooks: .tox/create-venvs.log
-	"./$(dir $(<))py36/bin/pre-commit" uninstall || true
-	"./$(dir $(<))py36/bin/pre-commit" uninstall -t pre-push || true
-
 .PHONY: test
 ## Run all tests and code reports
 test: .tox/create-venvs.log
@@ -215,7 +204,7 @@ clean:
 	rm -f *.pkg
 
 .PHONY: distclean
-distclean: clean uninstall-vcs-hooks
+distclean: clean
 	rm -rf src/*.egg-info
 	rm -f src/allmydata/_version.py
 	rm -f src/allmydata/_appname.py
@@ -261,6 +250,3 @@ src/allmydata/_version.py:
 
 .tox/create-venvs.log: tox.ini setup.py
 	tox --notest -p all | tee -a "$(@)"
-
-$(VCS_HOOKS): .tox/create-venvs.log .pre-commit-config.yaml
-	"./$(dir $(<))py36/bin/pre-commit" install --hook-type $(@:.git/hooks/%=%)
diff --git a/docs/developer-guide.rst b/docs/developer-guide.rst
index 2d26e68a4..a44414f8f 100644
--- a/docs/developer-guide.rst
+++ b/docs/developer-guide.rst
@@ -5,23 +5,17 @@ Developer Guide
 Pre-commit Checks
 -----------------
 
-This project is configured for use with `pre-commit`_ to install `VCS/git hooks`_ which
-perform some static code analysis checks and other code checks to catch common errors
-before each commit and to run the full self-test suite to find less obvious regressions
-before each push to a remote.
+This project is configured for use with `pre-commit`_ to install `VCS/git hooks`_ which perform some static code analysis checks and other code checks to catch common errors.
+These hooks can be configured to run before commits or pushes
 
 For example::
 
-  tahoe-lafs $ make install-vcs-hooks
-  ...
-  + ./.tox//py36/bin/pre-commit install --hook-type pre-commit
-  pre-commit installed at .git/hooks/pre-commit
-  + ./.tox//py36/bin/pre-commit install --hook-type pre-push
+  tahoe-lafs $ pre-commit install --hook-type pre-push
   pre-commit installed at .git/hooks/pre-push
-  tahoe-lafs $ python -c "import pathlib; pathlib.Path('src/allmydata/tabbed.py').write_text('def foo():\\n\\tpass\\n')"
-  tahoe-lafs $ git add src/allmydata/tabbed.py
+  tahoe-lafs $ echo "undefined" > src/allmydata/undefined_name.py
+  tahoe-lafs $ git add src/allmydata/undefined_name.py
   tahoe-lafs $ git commit -a -m "Add a file that violates flake8"
-  ...
+  tahoe-lafs $ git push
   codechecks...............................................................Failed
   - hook id: codechecks
   - exit code: 1
@@ -30,58 +24,17 @@ For example::
   codechecks inst-nodeps: ...
   codechecks installed: ...
   codechecks run-test-pre: PYTHONHASHSEED='...'
-  codechecks run-test: commands[0] | flake8 src static misc setup.py
-  src/allmydata/tabbed.py:2:1: W191 indentation contains tabs
-  ERROR: InvocationError for command ./tahoe-lafs/.tox/codechecks/bin/flake8 src static misc setup.py (exited with code 1)
+  codechecks run-test: commands[0] | flake8 src/allmydata/undefined_name.py
+  src/allmydata/undefined_name.py:1:1: F821 undefined name 'undefined'
+  ERROR: InvocationError for command ./tahoe-lafs/.tox/codechecks/bin/flake8 src/allmydata/undefined_name.py (exited with code 1)
   ___________________________________ summary ____________________________________
   ERROR:   codechecks: commands failed
-  ...
 
 To uninstall::
 
-  tahoe-lafs $ make uninstall-vcs-hooks
-  ...
-  + ./.tox/py36/bin/pre-commit uninstall
-  pre-commit uninstalled
-  + ./.tox/py36/bin/pre-commit uninstall -t pre-push
+  tahoe-lafs $ pre-commit uninstall --hook-type pre-push
   pre-push uninstalled
 
-Note that running the full self-test suite takes several minutes so expect pushing to
-take some time.  If you can't or don't want to wait for the hooks in some cases, use the
-``--no-verify`` option to ``$ git commit ...`` or ``$ git push ...``.  Alternatively,
-see the `pre-commit`_ documentation and CLI help output and use the committed
-`pre-commit configuration`_ as a starting point to write a local, uncommitted
-``../.pre-commit-config.local.yaml`` configuration to use instead.  For example::
-
-  tahoe-lafs $ ./.tox/py36/bin/pre-commit --help
-  tahoe-lafs $ ./.tox/py36/bin/pre-commit instll --help
-  tahoe-lafs $ cp  "./.pre-commit-config.yaml" "./.pre-commit-config.local.yaml"
-  tahoe-lafs $ editor "./.pre-commit-config.local.yaml"
-  ...
-  tahoe-lafs $ ./.tox/py36/bin/pre-commit install -c "./.pre-commit-config.local.yaml" -t pre-push
-  pre-commit installed at .git/hooks/pre-push
-  tahoe-lafs $ git commit -a -m "Add a file that violates flake8"
-  [3398.pre-commit 29f8f43d2] Add a file that violates flake8
-   1 file changed, 2 insertions(+)
-   create mode 100644 src/allmydata/tabbed.py
-  tahoe-lafs $ git push
-  ...
-  codechecks...............................................................Failed
-  - hook id: codechecks
-  - exit code: 1
-
-  GLOB sdist-make: ./tahoe-lafs/setup.py
-  codechecks inst-nodeps: ...
-  codechecks installed: ...
-  codechecks run-test-pre: PYTHONHASHSEED='...'
-  codechecks run-test: commands[0] | flake8 src static misc setup.py
-  src/allmydata/tabbed.py:2:1: W191 indentation contains tabs
-  ERROR: InvocationError for command ./tahoe-lafs/.tox/codechecks/bin/flake8 src static misc setup.py (exited with code 1)
-  ___________________________________ summary ____________________________________
-  ERROR:   codechecks: commands failed
-  ...
-
-  error: failed to push some refs to 'github.com:jaraco/tahoe-lafs.git'
 
 
 .. _`pre-commit`: https://pre-commit.com

From 96bee384c9fcb36c38aefefb922f9a81362c307a Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Sat, 5 Dec 2020 09:06:29 -0500
Subject: [PATCH 60/67] news fragment

---
 newsfragments/3547.minor | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 create mode 100644 newsfragments/3547.minor

diff --git a/newsfragments/3547.minor b/newsfragments/3547.minor
new file mode 100644
index 000000000..e69de29bb

From ac7491680be699a4aa2920efcfc6eba45493c6ab Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Sat, 5 Dec 2020 09:06:45 -0500
Subject: [PATCH 61/67] fix unicode/bytes stuff in the affected tests

---
 src/allmydata/scripts/common.py       | 4 +++-
 src/allmydata/test/test_introducer.py | 8 ++++----
 2 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/src/allmydata/scripts/common.py b/src/allmydata/scripts/common.py
index b20cca65f..29bb1d5f1 100644
--- a/src/allmydata/scripts/common.py
+++ b/src/allmydata/scripts/common.py
@@ -121,11 +121,13 @@ def write_introducer(basedir, petname, furl):
     Overwrite the node's ``introducers.yaml`` with a file containing the given
     introducer information.
     """
+    if isinstance(furl, bytes):
+        furl = furl.decode("utf-8")
     basedir.child(b"private").child(b"introducers.yaml").setContent(
         safe_dump({
             "introducers": {
                 petname: {
-                    "furl": furl.decode("ascii"),
+                    "furl": furl,
                 },
             },
         }).encode("ascii"),
diff --git a/src/allmydata/test/test_introducer.py b/src/allmydata/test/test_introducer.py
index 3aed3f049..2c5354fe0 100644
--- a/src/allmydata/test/test_introducer.py
+++ b/src/allmydata/test/test_introducer.py
@@ -818,8 +818,8 @@ class Announcements(AsyncTestCase):
         # until the introducer connection is established). To avoid getting
         # confused by this, disable storage.
         with basedir.child("tahoe.cfg").open("w") as f:
-            f.write("[storage]\n")
-            f.write("enabled = false\n")
+            f.write(b"[storage]\n")
+            f.write(b"enabled = false\n")
 
         c = yield create_client(basedir.path)
         ic = c.introducer_clients[0]
@@ -906,8 +906,8 @@ class ClientSeqnums(AsyncBrokenTestCase):
         # until the introducer connection is established). To avoid getting
         # confused by this, disable storage.
         with basedir.child("tahoe.cfg").open("w") as f:
-            f.write("[storage]\n")
-            f.write("enabled = false\n")
+            f.write(b"[storage]\n")
+            f.write(b"enabled = false\n")
 
         c = yield create_client(basedir.path)
         ic = c.introducer_clients[0]

From 386f30fa797b98de7fccf6a5c06960936849d27e Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Sun, 6 Dec 2020 10:30:08 -0500
Subject: [PATCH 62/67] Remove Mock from test_threshold_reached

---
 src/allmydata/test/no_network.py          |  3 +
 src/allmydata/test/test_storage_client.py | 88 ++++++++++++++++++++---
 2 files changed, 82 insertions(+), 9 deletions(-)

diff --git a/src/allmydata/test/no_network.py b/src/allmydata/test/no_network.py
index 54095f15d..f5f425dbc 100644
--- a/src/allmydata/test/no_network.py
+++ b/src/allmydata/test/no_network.py
@@ -89,6 +89,9 @@ class LocalWrapper(object):
         self.counter_by_methname = {}
         self._fireEventually = fireEventually
 
+    def getLocationHints(self):
+        return []
+
     def _clear_counters(self):
         self.counter_by_methname = {}
 
diff --git a/src/allmydata/test/test_storage_client.py b/src/allmydata/test/test_storage_client.py
index ed1dc30a9..db0b1467f 100644
--- a/src/allmydata/test/test_storage_client.py
+++ b/src/allmydata/test/test_storage_client.py
@@ -17,7 +17,6 @@ from json import (
 )
 
 import hashlib
-from mock import Mock
 from fixtures import (
     TempDir,
 )
@@ -44,12 +43,20 @@ from hyperlink import (
     URL,
 )
 
+import attr
+
+from twisted.internet.interfaces import (
+    IStreamClientEndpoint,
+)
 from twisted.application.service import (
     Service,
 )
 
 from twisted.trial import unittest
-from twisted.internet.defer import succeed, inlineCallbacks
+from twisted.internet.defer import (
+    Deferred,
+    inlineCallbacks,
+)
 from twisted.python.filepath import (
     FilePath,
 )
@@ -57,7 +64,11 @@ from twisted.python.filepath import (
 from foolscap.api import (
     Tub,
 )
+from foolscap.ipb import (
+    IConnectionHintHandler,
+)
 
+from .no_network import LocalWrapper
 from .common import (
     EMPTY_CLIENT_CONFIG,
     SyncTestCase,
@@ -84,6 +95,9 @@ from allmydata.storage_client import (
     _FoolscapStorage,
     _NullStorage,
 )
+from ..storage.server import (
+    StorageServer,
+)
 from allmydata.interfaces import (
     IConnectionStatus,
     IStorageServer,
@@ -529,6 +543,47 @@ def make_broker(tub_maker=None):
     return StorageFarmBroker(True, tub_maker, EMPTY_CLIENT_CONFIG)
 
 
+@implementer(IStreamClientEndpoint)
+@attr.s
+class SpyEndpoint(object):
+    """
+    Observe and record connection attempts.
+
+    :ivar list _append: A callable that accepts two-tuples.  For each
+        attempted connection, it will be called with ``Deferred`` that was
+        returned and the ``Factory`` that was passed in.
+    """
+    _append = attr.ib()
+
+    def connect(self, factory):
+        """
+        Record the connection attempt.
+
+        :return: A ``Deferred`` that ``SpyEndpoint`` will not fire.
+        """
+        d = Deferred()
+        self._append((d, factory))
+        return d
+
+
+@implementer(IConnectionHintHandler)
+@attr.s
+class SpyHandler(object):
+    """
+    A Foolscap connection hint handler for the "spy" hint type.  Connections
+    are handled by just observing and recording them.
+
+    :ivar list _connects: A list containing one element for each connection
+        attempted with this handler.  Each element is a two-tuple of the
+        ``Deferred`` that was returned from ``connect`` and the factory that
+        was passed to ``connect``.
+    """
+    _connects = attr.ib(default=attr.Factory(list))
+
+    def hint_to_endpoint(self, hint, reactor, update_status):
+        return (SpyEndpoint(self._connects.append), hint)
+
+
 class TestStorageFarmBroker(unittest.TestCase):
 
     def test_static_servers(self):
@@ -620,6 +675,11 @@ storage:
         def make_tub(*args, **kwargs):
             return new_tubs.pop()
         broker = make_broker(make_tub)
+        # Start the broker so that it will start Tubs attached to it so they
+        # will attempt to make connections as necessary so that we can observe
+        # those connections.
+        broker.startService()
+        self.addCleanup(broker.stopService)
         done = broker.when_connected_enough(5)
         broker.use_introducer(introducer)
         # subscribes to "storage" to learn of new storage nodes
@@ -637,15 +697,25 @@ storage:
         }
 
         def add_one_server(x):
-            data["anonymous-storage-FURL"] = b"pb://%s@nowhere/fake" % (base32.b2a(b"%d" % x),)
-            tub = Mock()
+            data["anonymous-storage-FURL"] = b"pb://%s@spy:nowhere/fake" % (base32.b2a(b"%d" % x),)
+            tub = Tub()
+            connects = []
+            spy = SpyHandler(connects)
+            tub.addConnectionHintHandler("spy", spy)
             new_tubs.append(tub)
             got_announcement(b'v0-1234-%d' % x, data)
-            self.assertEqual(tub.mock_calls[-1][0], 'connectTo')
-            got_connection = tub.mock_calls[-1][1][1]
-            rref = Mock()
-            rref.callRemote = Mock(return_value=succeed(1234))
-            got_connection(rref)
+
+            self.assertEqual(
+                1, len(connects),
+                "Expected one connection attempt, got {!r} instead".format(connects),
+            )
+
+            # Skip over all the Foolscap negotiation.  It's complex with lots
+            # of pieces and I don't want to figure out how to fake
+            # it. -exarkun
+            native = broker.servers[b"v0-1234-%d" % (x,)]
+            rref = LocalWrapper(StorageServer(self.mktemp(), b"x" * 20))
+            native._got_connection(rref)
 
         # first 4 shouldn't trigger connected_threashold
         for x in range(4):

From 8278fa9c620320b0627eb3a98910d68d1ac24aa3 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Sun, 6 Dec 2020 10:39:25 -0500
Subject: [PATCH 63/67] Name the pem so it gets installed

Also use the new_tub helper that reads the pem so we don't have to generate a
new cert in this test
---
 src/allmydata/test/data/{node.pem => node.pem.txt} | 0
 src/allmydata/test/test_storage_client.py          | 4 ++--
 2 files changed, 2 insertions(+), 2 deletions(-)
 rename src/allmydata/test/data/{node.pem => node.pem.txt} (100%)

diff --git a/src/allmydata/test/data/node.pem b/src/allmydata/test/data/node.pem.txt
similarity index 100%
rename from src/allmydata/test/data/node.pem
rename to src/allmydata/test/data/node.pem.txt
diff --git a/src/allmydata/test/test_storage_client.py b/src/allmydata/test/test_storage_client.py
index db0b1467f..6b89c5bf0 100644
--- a/src/allmydata/test/test_storage_client.py
+++ b/src/allmydata/test/test_storage_client.py
@@ -527,7 +527,7 @@ def new_tub():
     # Use a pre-generated key so the tests don't spend a lot of time
     # generating new ones.
     data = FilePath(__file__).sibling(b"data")
-    privkey = data.child(b"node.pem")
+    privkey = data.child(b"node.pem.txt")
     return Tub(
         certData=privkey.getContent(),
     )
@@ -698,7 +698,7 @@ storage:
 
         def add_one_server(x):
             data["anonymous-storage-FURL"] = b"pb://%s@spy:nowhere/fake" % (base32.b2a(b"%d" % x),)
-            tub = Tub()
+            tub = new_tub()
             connects = []
             spy = SpyHandler(connects)
             tub.addConnectionHintHandler("spy", spy)

From d83e4790c335bd0231021e04433abbf626961722 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Mon, 7 Dec 2020 13:14:00 -0500
Subject: [PATCH 64/67] Let subscribed_to get initialized right

---
 src/allmydata/test/test_storage_client.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/allmydata/test/test_storage_client.py b/src/allmydata/test/test_storage_client.py
index 531193804..913d9bf9c 100644
--- a/src/allmydata/test/test_storage_client.py
+++ b/src/allmydata/test/test_storage_client.py
@@ -669,7 +669,6 @@ storage:
             None,
             None,
             None,
-            None,
         )
         new_tubs = []
         def make_tub(*args, **kwargs):

From eddf7fd8f906b5903ad4c5cab67031ff78937489 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Tue, 8 Dec 2020 10:34:25 -0500
Subject: [PATCH 65/67] Remove the pre-generated key/certificate

---
 src/allmydata/test/data/node.pem.txt      | 45 -----------------------
 src/allmydata/test/test_storage_client.py | 12 +++---
 2 files changed, 5 insertions(+), 52 deletions(-)
 delete mode 100644 src/allmydata/test/data/node.pem.txt

diff --git a/src/allmydata/test/data/node.pem.txt b/src/allmydata/test/data/node.pem.txt
deleted file mode 100644
index dd0928cdd..000000000
--- a/src/allmydata/test/data/node.pem.txt
+++ /dev/null
@@ -1,45 +0,0 @@
------BEGIN CERTIFICATE-----
-MIICojCCAYoCAQEwDQYJKoZIhvcNAQELBQAwFzEVMBMGA1UEAwwMbmV3cGJfdGhp
-bmd5MB4XDTIwMDEwMjAxNDAzM1oXDTIxMDEwMTAxNDAzM1owFzEVMBMGA1UEAwwM
-bmV3cGJfdGhpbmd5MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA1iNV
-z07PYwZwucl87QlL2TFZvDxD4flZ/p3BZE3DCT5Efn9w2NT4sHXL1e+R/qsDFuNG
-bw1y1TRM0DGK6Wr0XRT2mLQULNgB8y/HrhcSdONsYRyWdj+LimyECKjwh0iSkApv
-Yj/7IOuq6dOoh67YXPdf75OHLShm4+8q8fuwhBL+nuuO4NhZDJKupYHcnuCkcF88
-LN77HKrrgbpyVmeghUkwJMLeJCewvYVlambgWRiuGGexFgAm6laS3rWetOcdm9eg
-FoA9PKNN6xvPatbj99MPoLpBbzsI64M0yT/wTSw1pj/Nom3rwfMa2OH8Kk7c8R/r
-U3xj4ZY1DTlGERvejQIDAQABMA0GCSqGSIb3DQEBCwUAA4IBAQAwyQjQ3ZgtJ3JW
-r3/EPdqSUBamTfXIpOh9rXmRjPpbe+MvenqIzl4q+GnkL5mdEb1e1hdKQZgFQ5Q5
-tbcNIz6h5C07KaNtbqhZCx5c/RUEH87VeXuAuOqZHbZWJ18q0tnk+YgWER2TOkgE
-RI2AslcsJBt88UUOjHX6/7J3KjPFaAjW1QV3TTsHxk14aYDYJwPdz+ijchgbOPQ0
-i+ilhzcB+qQnOC1s4xQSFo+zblTO7EgqM9KpupYfOVFh46P1Mak2W8EDvhz0livl
-OROXJ6nR/13lmQdfVX6T45d+ITBwtmW2nGAh3oI3JlArGKHaW+7qnuHR72q9FSES
-cEYA/wmk
------END CERTIFICATE-----
------BEGIN PRIVATE KEY-----
-MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDWI1XPTs9jBnC5
-yXztCUvZMVm8PEPh+Vn+ncFkTcMJPkR+f3DY1PiwdcvV75H+qwMW40ZvDXLVNEzQ
-MYrpavRdFPaYtBQs2AHzL8euFxJ042xhHJZ2P4uKbIQIqPCHSJKQCm9iP/sg66rp
-06iHrthc91/vk4ctKGbj7yrx+7CEEv6e647g2FkMkq6lgdye4KRwXzws3vscquuB
-unJWZ6CFSTAkwt4kJ7C9hWVqZuBZGK4YZ7EWACbqVpLetZ605x2b16AWgD08o03r
-G89q1uP30w+gukFvOwjrgzTJP/BNLDWmP82ibevB8xrY4fwqTtzxH+tTfGPhljUN
-OUYRG96NAgMBAAECggEAJ5xztBx0+nFnisZ9yG8uy6d4XPyc5gE1J4dRDdfgmyYc
-j3XNjx6ePi4cHZ/qVryVnrc+AS7wrgW1q9FuS81QFKPbFdZB4SW3/p85BbgY3uxu
-0Ovz3T3V9y4polx12eCP0/tKLVd+gdF2VTik9Sxfs5rC8VNN7wmJNuK4A/k15sgy
-BIu/R8NlMNGQySNhtccp+dzB8uTyKx5zFZhVvnAK/3YX9BC2V4QBW9JxO4S8N0/9
-48e9Sw/fGCfQ/EFPKGCvTvfuRqJ+4t5k10FygXJ+s+y70ifYi+aSsjJBuranbLJp
-g5TwhuKnTWs8Nth3YRLbcJL4VBIOehjAWy8pDMMtlQKBgQD0O8cHb8cOTGW0BijC
-NDofhA2GooQUUR3WL324PXWZq0DXuBDQhJVBKWO3AYonivhhd/qWO8lea9MEmU41
-nKZ7maS4B8AJLJC08P8GL1uCIE/ezEXEi9JwC1zJiyl595Ap4lSAozH0DwjNvmGL
-5mIdYg0BliqFXbloNJkNlb7INwKBgQDgdGEIWXc5Y1ncWNs6iDIV/t2MlL8vLrP0
-hpkl/QiMndOQyD6JBo0+ZqvOQTSS4NTSxBROjPxvFbEJ3eH8Pmn8gHOf46fzP1OJ
-wlYv0gYzkN4FE/tN6JnO2u9pN0euyyZLM1fnEcrMWColMN8JlWjtA7Gbxm8lkfa4
-3vicaJtlWwKBgQCQYL4ZgVR0+Wit8W4qz+EEPHYafvwBXqp6sXxqa7qXawtb+q3F
-9nqdGLCfwMNA+QA37ksugI1byfXmpBH902r/aiZbvAkj4zpwHH9F0r0PwbY1iSA9
-PkLahX0Gj8OnHFgWynsVyGOBWVnk9oSHxVt+7zWtGG5uhKdUGLPZugocJQKBgB61
-7bzduOFiRZ5PjhdxISE/UQL2Kz6Cbl7rt7Kp72yF/7eUnnHTMqoyFBnRdCcQmi4I
-ZBrnUXbFigamlFAWHhxNWwSqeoVeychUjcRXQT/291nMhRsA02KpNA66YJV6+E9b
-xBA6r/vLqGCUUkAWcFfVpIyC1xxV32MmJvAHpBN3AoGAPF3MUFiO0iKNZfst6Tm3
-rzrldLawDo98DRZ7Yb2kWlWZYqUk/Nvryvo2cns75WGSMDYVbbRp+BY7kZmNYa9K
-iQzKDL54ZRu6V+getJdeAO8yXoCmnZKxt5OHvOSrQMfAmFKSwLwxBbZBfXEyuune
-yfusXLtCgajpreoVIa0xWdQ=
------END PRIVATE KEY-----
diff --git a/src/allmydata/test/test_storage_client.py b/src/allmydata/test/test_storage_client.py
index 913d9bf9c..44e481d52 100644
--- a/src/allmydata/test/test_storage_client.py
+++ b/src/allmydata/test/test_storage_client.py
@@ -520,17 +520,15 @@ class StoragePluginWebPresence(AsyncTestCase):
         )
 
 
+_aCertPEM = Tub().myCertificate.dumpPEM()
 def new_tub():
     """
     Make a new ``Tub`` with a hard-coded private key.
     """
-    # Use a pre-generated key so the tests don't spend a lot of time
-    # generating new ones.
-    data = FilePath(__file__).sibling(b"data")
-    privkey = data.child(b"node.pem.txt")
-    return Tub(
-        certData=privkey.getContent(),
-    )
+    # Use a private key / certificate generated by Tub how it wants.  But just
+    # re-use the same one every time so we don't waste a lot of time
+    # generating them over and over in the tests.
+    return Tub(certData=_aCertPEM)
 
 
 def make_broker(tub_maker=None):

From 69d3dad64691041161e0132925ccac89a939f21d Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Tue, 8 Dec 2020 10:37:31 -0500
Subject: [PATCH 66/67] Get rid of remote_host / get_remote_host

Nothing uses it and if we don't provide it we don't need getLocationHints
---
 src/allmydata/interfaces.py           | 1 -
 src/allmydata/storage_client.py       | 6 ------
 src/allmydata/test/no_network.py      | 3 ---
 src/allmydata/test/test_introducer.py | 2 --
 4 files changed, 12 deletions(-)

diff --git a/src/allmydata/interfaces.py b/src/allmydata/interfaces.py
index 49dcf7646..b9a757b08 100644
--- a/src/allmydata/interfaces.py
+++ b/src/allmydata/interfaces.py
@@ -521,7 +521,6 @@ class IStorageBroker(Interface):
           oldest_supported: the peer's oldest supported version, same
 
           rref: the RemoteReference, if connected, otherwise None
-          remote_host: the IAddress, if connected, otherwise None
 
         This method is intended for monitoring interfaces, such as a web page
         that describes connecting and connected peers.
diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py
index 823c15b7a..87560e51c 100644
--- a/src/allmydata/storage_client.py
+++ b/src/allmydata/storage_client.py
@@ -693,7 +693,6 @@ class NativeStorageServer(service.MultiService):
     @ivar nickname: the server's self-reported nickname (unicode), same
 
     @ivar rref: the RemoteReference, if connected, otherwise None
-    @ivar remote_host: the IAddress, if connected, otherwise None
     """
 
     VERSION_DEFAULTS = UnicodeKeyDict({
@@ -719,7 +718,6 @@ class NativeStorageServer(service.MultiService):
 
         self.last_connect_time = None
         self.last_loss_time = None
-        self.remote_host = None
         self._rref = None
         self._is_connected = False
         self._reconnector = None
@@ -828,8 +826,6 @@ class NativeStorageServer(service.MultiService):
         return None
     def get_announcement(self):
         return self.announcement
-    def get_remote_host(self):
-        return self.remote_host
 
     def get_connection_status(self):
         last_received = None
@@ -877,7 +873,6 @@ class NativeStorageServer(service.MultiService):
                 level=log.NOISY, parent=lp)
 
         self.last_connect_time = time.time()
-        self.remote_host = rref.getLocationHints()
         self._rref = rref
         self._is_connected = True
         rref.notifyOnDisconnect(self._lost)
@@ -903,7 +898,6 @@ class NativeStorageServer(service.MultiService):
         # get_connected_servers() or get_servers_for_psi()) can continue to
         # use s.get_rref().callRemote() and not worry about it being None.
         self._is_connected = False
-        self.remote_host = None
 
     def stop_connecting(self):
         # used when this descriptor has been superceded by another
diff --git a/src/allmydata/test/no_network.py b/src/allmydata/test/no_network.py
index ad7659704..59ab807bb 100644
--- a/src/allmydata/test/no_network.py
+++ b/src/allmydata/test/no_network.py
@@ -89,9 +89,6 @@ class LocalWrapper(object):
         self.counter_by_methname = {}
         self._fireEventually = fireEventually
 
-    def getLocationHints(self):
-        return []
-
     def _clear_counters(self):
         self.counter_by_methname = {}
 
diff --git a/src/allmydata/test/test_introducer.py b/src/allmydata/test/test_introducer.py
index 2c5354fe0..5350335e0 100644
--- a/src/allmydata/test/test_introducer.py
+++ b/src/allmydata/test/test_introducer.py
@@ -744,8 +744,6 @@ class SystemTest(SystemTestMixin, AsyncTestCase):
 class FakeRemoteReference(object):
     def notifyOnDisconnect(self, *args, **kwargs): pass
     def getRemoteTubID(self): return "62ubehyunnyhzs7r6vdonnm2hpi52w6y"
-    def getLocationHints(self): return ["tcp:here.example.com:1234",
-                                        "tcp:there.example.com2345"]
     def getPeer(self): return address.IPv4Address("TCP", "remote.example.com",
                                                   3456)
 

From 754b88a0b9cb9497d24c9f49430951d3bd452d2e Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Tue, 8 Dec 2020 13:47:53 -0500
Subject: [PATCH 67/67] Fix failing test.

---
 src/allmydata/test/common.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/allmydata/test/common.py b/src/allmydata/test/common.py
index 797e3dab9..f93272540 100644
--- a/src/allmydata/test/common.py
+++ b/src/allmydata/test/common.py
@@ -214,7 +214,7 @@ class UseNode(object):
 
     :ivar FilePath basedir: The base directory of the node.
 
-    :ivar bytes introducer_furl: The introducer furl with which to
+    :ivar str introducer_furl: The introducer furl with which to
         configure the client.
 
     :ivar dict[bytes, bytes] node_config: Configuration items for the *node*
@@ -225,7 +225,8 @@ class UseNode(object):
     plugin_config = attr.ib()
     storage_plugin = attr.ib()
     basedir = attr.ib(validator=attr.validators.instance_of(FilePath))
-    introducer_furl = attr.ib(validator=attr.validators.instance_of(bytes))
+    introducer_furl = attr.ib(validator=attr.validators.instance_of(str),
+                              converter=six.ensure_str)
     node_config = attr.ib(default=attr.Factory(dict))
 
     config = attr.ib(default=None)