From 00e856fed5a3447a60bc575cd2b75118a39d2770 Mon Sep 17 00:00:00 2001
From: "Jason R. Coombs" <jaraco@jaraco.com>
Date: Sat, 30 Jan 2021 11:00:19 -0500
Subject: [PATCH 01/50] Declare scripts.types_ as ported (never had Python 2
 support).

---
 src/allmydata/scripts/types_.py | 2 ++
 src/allmydata/util/_python3.py  | 1 +
 2 files changed, 3 insertions(+)

diff --git a/src/allmydata/scripts/types_.py b/src/allmydata/scripts/types_.py
index 3937cb803..217dc2d28 100644
--- a/src/allmydata/scripts/types_.py
+++ b/src/allmydata/scripts/types_.py
@@ -1,3 +1,5 @@
+# Python 3 only
+
 from typing import List, Tuple, Type, Sequence, Any
 from allmydata.scripts.common import BaseOptions
 
diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py
index 38d0f4d7e..b5acd989d 100644
--- a/src/allmydata/util/_python3.py
+++ b/src/allmydata/util/_python3.py
@@ -72,6 +72,7 @@ PORTED_MODULES = [
     "allmydata.mutable.servermap",
     "allmydata.node",
     "allmydata.nodemaker",
+    "allmydata.scripts.types_",
     "allmydata.stats",
     "allmydata.storage_client",
     "allmydata.storage.common",

From 5bf2b09b81b28ca16cf6fbfb569057ec27843254 Mon Sep 17 00:00:00 2001
From: "Jason R. Coombs" <jaraco@jaraco.com>
Date: Sat, 30 Jan 2021 11:12:01 -0500
Subject: [PATCH 02/50] In scripts.runner, replace hard failure on Python 3
 with 'experimental' warning. Ref #3603. Closes #3035.

---
 src/allmydata/scripts/runner.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/allmydata/scripts/runner.py b/src/allmydata/scripts/runner.py
index 9a632a57d..af8b89d79 100644
--- a/src/allmydata/scripts/runner.py
+++ b/src/allmydata/scripts/runner.py
@@ -1,5 +1,6 @@
 from __future__ import print_function
 
+import warnings
 import os, sys
 from six.moves import StringIO
 import six
@@ -177,9 +178,8 @@ def _maybe_enable_eliot_logging(options, reactor):
     return options
 
 def run():
-    # TODO(3035): Remove tox-check when error becomes a warning
-    if 'TOX_ENV_NAME' not in os.environ:
-        assert sys.version_info < (3,), u"Tahoe-LAFS does not run under Python 3. Please use Python 2.7.x."
+    if not six.PY2:
+        warnings.warn("Support for Python 3 is experimental. Use at your own risk.")
 
     if sys.platform == "win32":
         from allmydata.windows.fixups import initialize

From 2bd244dde95d32a0949abfd465d06428b56179d9 Mon Sep 17 00:00:00 2001
From: "Jason R. Coombs" <jaraco@jaraco.com>
Date: Sat, 30 Jan 2021 11:24:06 -0500
Subject: [PATCH 03/50] Declare scripts.runner as ported. Ref #3603.

---
 src/allmydata/util/_python3.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py
index b5acd989d..242bdaae8 100644
--- a/src/allmydata/util/_python3.py
+++ b/src/allmydata/util/_python3.py
@@ -72,6 +72,7 @@ PORTED_MODULES = [
     "allmydata.mutable.servermap",
     "allmydata.node",
     "allmydata.nodemaker",
+    "allmydata.scripts.runner",
     "allmydata.scripts.types_",
     "allmydata.stats",
     "allmydata.storage_client",

From 210eb5b5293f4bd4d6734a2a4317484fd26215e8 Mon Sep 17 00:00:00 2001
From: "Jason R. Coombs" <jaraco@jaraco.com>
Date: Sat, 30 Jan 2021 11:32:12 -0500
Subject: [PATCH 04/50] Add test for escape_path.

---
 src/allmydata/scripts/common.py | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/src/allmydata/scripts/common.py b/src/allmydata/scripts/common.py
index d73344274..e50617fd7 100644
--- a/src/allmydata/scripts/common.py
+++ b/src/allmydata/scripts/common.py
@@ -1,3 +1,5 @@
+# coding: utf-8
+
 from __future__ import print_function
 
 import os, sys, urllib, textwrap
@@ -270,6 +272,16 @@ def get_alias(aliases, path_unicode, default):
     return uri.from_string_dirnode(aliases[alias]).to_string(), path[colon+1:]
 
 def escape_path(path):
+    # type: (str) -> str
+    """
+    Return path quoted to US-ASCII.
+
+    >>> path = u'/føö/bar/☃'
+    >>> escape_path(path)
+    '/f%C3%B8%C3%B6/bar/%E2%98%83'
+    >>> escape_path(path).encode('ascii')
+    b'/f%C3%B8%C3%B6/bar/%E2%98%83'
+    """
     # this always returns bytes, specifically US-ASCII, valid URL characters
     segments = path.split("/")
     return "/".join([urllib.quote(unicode_to_url(s)) for s in segments])

From 5fc9674d3a55d0e9591fdce8307b865ae0256e5d Mon Sep 17 00:00:00 2001
From: "Jason R. Coombs" <jaraco@jaraco.com>
Date: Sat, 30 Jan 2021 11:47:53 -0500
Subject: [PATCH 05/50] Update escape_path for Python 3 compatibility.

---
 src/allmydata/scripts/common.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/allmydata/scripts/common.py b/src/allmydata/scripts/common.py
index e50617fd7..ae9d78db7 100644
--- a/src/allmydata/scripts/common.py
+++ b/src/allmydata/scripts/common.py
@@ -2,10 +2,12 @@
 
 from __future__ import print_function
 
-import os, sys, urllib, textwrap
+import os, sys, textwrap
 import codecs
 from os.path import join
 
+from six.moves import urllib  # import urllib.parse
+
 try:
     from typing import Optional
     from .types_ import Parameters
@@ -284,4 +286,4 @@ def escape_path(path):
     """
     # this always returns bytes, specifically US-ASCII, valid URL characters
     segments = path.split("/")
-    return "/".join([urllib.quote(unicode_to_url(s)) for s in segments])
+    return "/".join([urllib.parse.quote(unicode_to_url(s)) for s in segments])

From 36b7fdaecf7cd3bfcc410b64c07007bcc61d6a5d Mon Sep 17 00:00:00 2001
From: "Jason R. Coombs" <jaraco@jaraco.com>
Date: Sat, 30 Jan 2021 12:15:36 -0500
Subject: [PATCH 06/50] Apply futurize to create_node.

---
 src/allmydata/scripts/create_node.py | 19 +++++++++++++------
 1 file changed, 13 insertions(+), 6 deletions(-)

diff --git a/src/allmydata/scripts/create_node.py b/src/allmydata/scripts/create_node.py
index 0f507f518..2d949b1ed 100644
--- a/src/allmydata/scripts/create_node.py
+++ b/src/allmydata/scripts/create_node.py
@@ -1,4 +1,11 @@
 from __future__ import print_function
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import unicode_literals
+
+from future.utils import PY2
+if PY2:
+    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
 import os
 import json
@@ -301,13 +308,13 @@ def write_node_config(c, config):
 
     if tor_config:
         c.write("[tor]\n")
-        for key, value in tor_config.items():
+        for key, value in list(tor_config.items()):
             c.write("%s = %s\n" % (key, value))
         c.write("\n")
 
     if i2p_config:
         c.write("[i2p]\n")
-        for key, value in i2p_config.items():
+        for key, value in list(i2p_config.items()):
             c.write("%s = %s\n" % (key, value))
         c.write("\n")
 
@@ -370,7 +377,7 @@ def _get_config_via_wormhole(config):
         relay_url=relay_url,
         reactor=reactor,
     )
-    code = unicode(config['join'])
+    code = str(config['join'])
     wh.set_code(code)
     yield wh.get_welcome()
     print("Connected to wormhole server", file=out)
@@ -402,7 +409,7 @@ def create_node(config):
     err = config.stderr
     basedir = config['basedir']
     # This should always be called with an absolute Unicode basedir.
-    precondition(isinstance(basedir, unicode), basedir)
+    precondition(isinstance(basedir, str), basedir)
 
     if os.path.exists(basedir):
         if listdir_unicode(basedir):
@@ -437,7 +444,7 @@ def create_node(config):
             v = remote_config.get(k, None)
             if v is not None:
                 # we're faking usually argv-supplied options :/
-                if isinstance(v, unicode):
+                if isinstance(v, str):
                     v = v.encode(get_io_encoding())
                 config[k] = v
                 if k not in sensitive_keys:
@@ -475,7 +482,7 @@ def create_introducer(config):
     err = config.stderr
     basedir = config['basedir']
     # This should always be called with an absolute Unicode basedir.
-    precondition(isinstance(basedir, unicode), basedir)
+    precondition(isinstance(basedir, str), basedir)
 
     if os.path.exists(basedir):
         if listdir_unicode(basedir):

From d6082d853a0babdf570b54123cf0a8c78023ec3e Mon Sep 17 00:00:00 2001
From: "Jason R. Coombs" <jaraco@jaraco.com>
Date: Sat, 30 Jan 2021 12:23:16 -0500
Subject: [PATCH 07/50] Declare scripts.create_node as ported.

---
 src/allmydata/scripts/create_node.py | 2 ++
 src/allmydata/util/_python3.py       | 1 +
 2 files changed, 3 insertions(+)

diff --git a/src/allmydata/scripts/create_node.py b/src/allmydata/scripts/create_node.py
index 2d949b1ed..782bccda6 100644
--- a/src/allmydata/scripts/create_node.py
+++ b/src/allmydata/scripts/create_node.py
@@ -1,3 +1,5 @@
+# Ported to Python 3
+
 from __future__ import print_function
 from __future__ import absolute_import
 from __future__ import division
diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py
index 242bdaae8..e7d8e2049 100644
--- a/src/allmydata/util/_python3.py
+++ b/src/allmydata/util/_python3.py
@@ -72,6 +72,7 @@ PORTED_MODULES = [
     "allmydata.mutable.servermap",
     "allmydata.node",
     "allmydata.nodemaker",
+    "allmydata.scripts.create_node",
     "allmydata.scripts.runner",
     "allmydata.scripts.types_",
     "allmydata.stats",

From 57cb88638a8ee0782f9fcc5256412a93dc3c05ee Mon Sep 17 00:00:00 2001
From: "Jason R. Coombs" <jaraco@jaraco.com>
Date: Sat, 30 Jan 2021 12:46:44 -0500
Subject: [PATCH 08/50] In scripts.create_node, set the encoding once and write
 text.

---
 src/allmydata/scripts/create_node.py | 20 +++++++++++---------
 1 file changed, 11 insertions(+), 9 deletions(-)

diff --git a/src/allmydata/scripts/create_node.py b/src/allmydata/scripts/create_node.py
index 782bccda6..3afbeeb0d 100644
--- a/src/allmydata/scripts/create_node.py
+++ b/src/allmydata/scripts/create_node.py
@@ -9,6 +9,7 @@ from future.utils import PY2
 if PY2:
     from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
+import io
 import os
 import json
 
@@ -234,7 +235,7 @@ class CreateIntroducerOptions(NoDefaultBasedirOptions):
 @defer.inlineCallbacks
 def write_node_config(c, config):
     # this is shared between clients and introducers
-    c.write("# -*- mode: conf; coding: utf-8 -*-\n")
+    c.write("# -*- mode: conf; coding: {c.encoding} -*-\n".format(c=c))
     c.write("\n")
     c.write("# This file controls the configuration of the Tahoe node that\n")
     c.write("# lives in this directory. It is only read at node startup.\n")
@@ -253,7 +254,7 @@ def write_node_config(c, config):
 
     c.write("[node]\n")
     nickname = argv_to_unicode(config.get("nickname") or "")
-    c.write("nickname = %s\n" % (nickname.encode('utf-8'),))
+    c.write("nickname = %s\n" % (nickname,))
     if config["hide-ip"]:
         c.write("reveal-IP-address = false\n")
     else:
@@ -263,7 +264,7 @@ def write_node_config(c, config):
     webport = argv_to_unicode(config.get("webport") or "none")
     if webport.lower() == "none":
         webport = ""
-    c.write("web.port = %s\n" % (webport.encode('utf-8'),))
+    c.write("web.port = %s\n" % (webport,))
     c.write("web.static = public_html\n")
 
     listeners = config['listen'].split(",")
@@ -288,15 +289,14 @@ def write_node_config(c, config):
             tub_locations.append(i2p_location)
         if "tcp" in listeners:
             if config["port"]: # --port/--location are a pair
-                tub_ports.append(config["port"].encode('utf-8'))
-                tub_locations.append(config["location"].encode('utf-8'))
+                tub_ports.append(config["port"])
+                tub_locations.append(config["location"])
             else:
                 assert "hostname" in config
                 hostname = config["hostname"]
                 new_port = iputil.allocate_tcp_port()
                 tub_ports.append("tcp:%s" % new_port)
-                tub_locations.append("tcp:%s:%s" % (hostname.encode('utf-8'),
-                                                    new_port))
+                tub_locations.append("tcp:%s:%s" % (hostname, new_port))
         c.write("tub.port = %s\n" % ",".join(tub_ports))
         c.write("tub.location = %s\n" % ",".join(tub_locations))
     c.write("\n")
@@ -456,7 +456,8 @@ def create_node(config):
                     print("  {}: [sensitive data; see tahoe.cfg]".format(k), file=out)
 
     fileutil.make_dirs(os.path.join(basedir, "private"), 0o700)
-    with open(os.path.join(basedir, "tahoe.cfg"), "w") as c:
+    cfg_name = os.path.join(basedir, "tahoe.cfg")
+    with io.open(cfg_name, "w", encoding='utf-8') as c:
         yield write_node_config(c, config)
         write_client_config(c, config)
 
@@ -498,7 +499,8 @@ def create_introducer(config):
     write_tac(basedir, "introducer")
 
     fileutil.make_dirs(os.path.join(basedir, "private"), 0o700)
-    with open(os.path.join(basedir, "tahoe.cfg"), "w") as c:
+    cfg_name = os.path.join(basedir, "tahoe.cfg")
+    with io.open(cfg_name, "w", encoding='utf-8') as c:
         yield write_node_config(c, config)
 
     print("Introducer created in %s" % quote_local_unicode_path(basedir), file=out)

From 621ae58abea737221adc0282859794ceaa42244e Mon Sep 17 00:00:00 2001
From: "Jason R. Coombs" <jaraco@jaraco.com>
Date: Sun, 31 Jan 2021 11:13:59 -0500
Subject: [PATCH 09/50] Avoid overthinking encoding when handling usage errors.
 'test_unicode_arguments_and_output' still passes on Python 2. Ref #3603.

---
 src/allmydata/scripts/runner.py | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/src/allmydata/scripts/runner.py b/src/allmydata/scripts/runner.py
index af8b89d79..705fa37bf 100644
--- a/src/allmydata/scripts/runner.py
+++ b/src/allmydata/scripts/runner.py
@@ -16,7 +16,7 @@ from twisted.internet import defer, task, threads
 from allmydata.scripts.common import get_default_nodedir
 from allmydata.scripts import debug, create_node, cli, \
     admin, tahoe_run, tahoe_invite
-from allmydata.util.encodingutil import quote_output, quote_local_unicode_path, get_io_encoding
+from allmydata.util.encodingutil import quote_local_unicode_path
 from allmydata.util.eliotutil import (
     opt_eliot_destination,
     opt_help_eliot_destinations,
@@ -121,11 +121,7 @@ def parse_or_exit_with_explanation(argv, stdout=sys.stdout):
         while hasattr(c, 'subOptions'):
             c = c.subOptions
         print(str(c), file=stdout)
-        try:
-            msg = e.args[0].decode(get_io_encoding())
-        except Exception:
-            msg = repr(e)
-        print("%s:  %s\n" % (sys.argv[0], quote_output(msg, quotemarks=False)), file=stdout)
+        print("%s:  %s\n" % (sys.argv[0], e), file=stdout)
         sys.exit(1)
     return config
 

From 7ed3d9597eea55ff9eb17de84a0ff2507c49a560 Mon Sep 17 00:00:00 2001
From: "Jason R. Coombs" <jaraco@jaraco.com>
Date: Fri, 12 Feb 2021 16:17:44 -0500
Subject: [PATCH 10/50] Add newsfragment

---
 newsfragments/3603.minor.rst | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 newsfragments/3603.minor.rst

diff --git a/newsfragments/3603.minor.rst b/newsfragments/3603.minor.rst
new file mode 100644
index 000000000..512ad0194
--- /dev/null
+++ b/newsfragments/3603.minor.rst
@@ -0,0 +1 @@
+Ported allmydata.scripts.create_node and .runner to Python 3. Declared scripts.types_ as Python 3 only.

From eefb7004c902d41ed08a9e42b14f56e634fdf9fd Mon Sep 17 00:00:00 2001
From: "Jason R. Coombs" <jaraco@jaraco.com>
Date: Mon, 15 Feb 2021 09:20:45 -0500
Subject: [PATCH 11/50] Add test_runner to the ported test modules. Selectively
 decode sys.argv on Python 2 only. Fixes 6 test failures on Python 3.

---
 src/allmydata/test/test_runner.py | 9 +++++++--
 src/allmydata/util/_python3.py    | 1 +
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/src/allmydata/test/test_runner.py b/src/allmydata/test/test_runner.py
index cf56e8baa..d8f9387c0 100644
--- a/src/allmydata/test/test_runner.py
+++ b/src/allmydata/test/test_runner.py
@@ -10,6 +10,8 @@ from eliot import (
     log_call,
 )
 
+import six
+
 from twisted.trial import unittest
 
 from twisted.internet import reactor
@@ -23,7 +25,7 @@ from twisted.python.runtime import (
     platform,
 )
 from allmydata.util import fileutil, pollmixin
-from allmydata.util.encodingutil import unicode_to_argv, get_filesystem_encoding
+from allmydata.util.encodingutil import unicode_to_argv
 from allmydata.test import common_util
 import allmydata
 from .common import (
@@ -72,7 +74,10 @@ def run_bintahoe(extra_argv, python_options=None):
     :return: A three-tuple of stdout (unicode), stderr (unicode), and the
         child process "returncode" (int).
     """
-    argv = [sys.executable.decode(get_filesystem_encoding())]
+    # fixme: below, 'unicode_to_argv' is called so ensure that
+    # executable is unicode to support that expectation.
+    executable = sys.executable.decode('utf-8') if six.PY2 else sys.executable
+    argv = [executable]
     if python_options is not None:
         argv.extend(python_options)
     argv.extend([u"-m", u"allmydata.scripts.runner"])
diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py
index a331a3734..79622650c 100644
--- a/src/allmydata/util/_python3.py
+++ b/src/allmydata/util/_python3.py
@@ -174,6 +174,7 @@ PORTED_TEST_MODULES = [
     "allmydata.test.test_pipeline",
     "allmydata.test.test_python3",
     "allmydata.test.test_repairer",
+    "allmydata.test.test_runner",
     "allmydata.test.test_sftp",
     "allmydata.test.test_spans",
     "allmydata.test.test_statistics",

From dc883c0440f930ab9f01db1ca1c7db75c69cf7d0 Mon Sep 17 00:00:00 2001
From: "Jason R. Coombs" <jaraco@jaraco.com>
Date: Mon, 15 Feb 2021 09:24:16 -0500
Subject: [PATCH 12/50] Decode config file on read. Fixes two test failures on
 Python 3.

---
 src/allmydata/test/test_runner.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/allmydata/test/test_runner.py b/src/allmydata/test/test_runner.py
index d8f9387c0..cca1ae70f 100644
--- a/src/allmydata/test/test_runner.py
+++ b/src/allmydata/test/test_runner.py
@@ -184,7 +184,7 @@ class CreateNode(unittest.TestCase):
             # 'create-node', and disabled for 'create-client'.
             tahoe_cfg = os.path.join(n1, "tahoe.cfg")
             self.failUnless(os.path.exists(tahoe_cfg))
-            content = fileutil.read(tahoe_cfg).replace('\r\n', '\n')
+            content = fileutil.read(tahoe_cfg).decode('utf-8').replace('\r\n', '\n')
             if kind == "client":
                 self.failUnless(re.search(r"\n\[storage\]\n#.*\nenabled = false\n", content), content)
             else:

From 3acad6544e56878513e33862d426b403d19ad25f Mon Sep 17 00:00:00 2001
From: "Jason R. Coombs" <jaraco@jaraco.com>
Date: Mon, 15 Feb 2021 09:28:14 -0500
Subject: [PATCH 13/50] Decode config file on read and expect bytes in the
 stdout. Fixes an additional test on Python 3.

---
 src/allmydata/test/test_runner.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/allmydata/test/test_runner.py b/src/allmydata/test/test_runner.py
index cca1ae70f..bee35522b 100644
--- a/src/allmydata/test/test_runner.py
+++ b/src/allmydata/test/test_runner.py
@@ -374,7 +374,7 @@ class RunNode(common_util.SignalMixin, unittest.TestCase, pollmixin.PollMixin):
         self.failUnlessEqual(returncode, 0)
 
         # Check that the --webport option worked.
-        config = fileutil.read(tahoe.config_file.path)
+        config = fileutil.read(tahoe.config_file.path).decode('utf-8')
         self.assertIn(
             '{}web.port = 0{}'.format(linesep, linesep),
             config,
@@ -387,7 +387,7 @@ class RunNode(common_util.SignalMixin, unittest.TestCase, pollmixin.PollMixin):
         # This will run until we stop it.
         tahoe.run(on_stdout(p))
         # Wait for startup to have proceeded to a reasonable point.
-        yield p.expect("client running")
+        yield p.expect(b"client running")
         tahoe.active()
 
         # read the storage.furl file so we can check that its contents don't
@@ -406,7 +406,7 @@ class RunNode(common_util.SignalMixin, unittest.TestCase, pollmixin.PollMixin):
         # We don't have to add another cleanup for this one, the one from
         # above is still registered.
         tahoe.run(on_stdout(p))
-        yield p.expect("client running")
+        yield p.expect(b"client running")
         tahoe.active()
 
         self.assertEqual(

From f183be9d6a859b4600d71f6564d851e663c23a97 Mon Sep 17 00:00:00 2001
From: "Jason R. Coombs" <jaraco@jaraco.com>
Date: Mon, 15 Feb 2021 09:33:54 -0500
Subject: [PATCH 14/50] Decode config file on read and expect bytes in the
 stdout. Fixes an additional test on Python 3.

---
 src/allmydata/test/test_runner.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/allmydata/test/test_runner.py b/src/allmydata/test/test_runner.py
index bee35522b..885af5efb 100644
--- a/src/allmydata/test/test_runner.py
+++ b/src/allmydata/test/test_runner.py
@@ -296,7 +296,7 @@ class RunNode(common_util.SignalMixin, unittest.TestCase, pollmixin.PollMixin):
 
         # This makes sure that node.url is written, which allows us to
         # detect when the introducer restarts in _node_has_restarted below.
-        config = fileutil.read(tahoe.config_file.path)
+        config = fileutil.read(tahoe.config_file.path).decode('utf-8')
         self.assertIn('{}web.port = {}'.format(linesep, linesep), config)
         fileutil.write(
             tahoe.config_file.path,
@@ -308,7 +308,7 @@ class RunNode(common_util.SignalMixin, unittest.TestCase, pollmixin.PollMixin):
 
         p = Expect()
         tahoe.run(on_stdout(p))
-        yield p.expect("introducer running")
+        yield p.expect(b"introducer running")
         tahoe.active()
 
         yield self.poll(tahoe.introducer_furl_file.exists)
@@ -331,7 +331,7 @@ class RunNode(common_util.SignalMixin, unittest.TestCase, pollmixin.PollMixin):
 
         p = Expect()
         tahoe.run(on_stdout(p))
-        yield p.expect("introducer running")
+        yield p.expect(b"introducer running")
 
         # Again, the second incarnation of the node might not be ready yet, so
         # poll until it is. This time introducer_furl_file already exists, so

From 30f5c71a499f1fbaa48d40aaa08ae0554b7da202 Mon Sep 17 00:00:00 2001
From: "Jason R. Coombs" <jaraco@jaraco.com>
Date: Mon, 15 Feb 2021 09:36:07 -0500
Subject: [PATCH 15/50] Encode expected message. Fixes remaining test failures
 in test_runner.

---
 src/allmydata/test/test_runner.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/allmydata/test/test_runner.py b/src/allmydata/test/test_runner.py
index 885af5efb..597a75fe8 100644
--- a/src/allmydata/test/test_runner.py
+++ b/src/allmydata/test/test_runner.py
@@ -497,7 +497,7 @@ class RunNode(common_util.SignalMixin, unittest.TestCase, pollmixin.PollMixin):
         client_running = p.expect(b"client running")
 
         result, index = yield DeferredList([
-            p.expect(expected_message),
+            p.expect(expected_message.encode('utf-8')),
             client_running,
         ], fireOnOneCallback=True, consumeErrors=True,
         )

From 3c93605ead266dde837324fa317843b419fd4778 Mon Sep 17 00:00:00 2001
From: "Jason R. Coombs" <jaraco@jaraco.com>
Date: Mon, 15 Feb 2021 12:15:41 -0500
Subject: [PATCH 16/50] Add docstring to types_.

---
 src/allmydata/scripts/types_.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/allmydata/scripts/types_.py b/src/allmydata/scripts/types_.py
index 217dc2d28..289e674ce 100644
--- a/src/allmydata/scripts/types_.py
+++ b/src/allmydata/scripts/types_.py
@@ -1,3 +1,7 @@
+"""
+Type definitions used by modules in this package.
+"""
+
 # Python 3 only
 
 from typing import List, Tuple, Type, Sequence, Any

From 2dc0a55d5d85fd9e9a41de1d9565ff978c5318c7 Mon Sep 17 00:00:00 2001
From: "Jason R. Coombs" <jaraco@jaraco.com>
Date: Mon, 15 Feb 2021 12:18:10 -0500
Subject: [PATCH 17/50] Truncate newsfragment

---
 newsfragments/3603.minor.rst | 1 -
 1 file changed, 1 deletion(-)

diff --git a/newsfragments/3603.minor.rst b/newsfragments/3603.minor.rst
index 512ad0194..e69de29bb 100644
--- a/newsfragments/3603.minor.rst
+++ b/newsfragments/3603.minor.rst
@@ -1 +0,0 @@
-Ported allmydata.scripts.create_node and .runner to Python 3. Declared scripts.types_ as Python 3 only.

From d59d64d6bdd087fba765129b2e0496ebff77dc98 Mon Sep 17 00:00:00 2001
From: "Jason R. Coombs" <jaraco@jaraco.com>
Date: Mon, 15 Feb 2021 12:38:37 -0500
Subject: [PATCH 18/50] Fully port test_runner. Introduces three new errors on
 Python 2.

---
 src/allmydata/test/test_runner.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/src/allmydata/test/test_runner.py b/src/allmydata/test/test_runner.py
index 597a75fe8..12b658a71 100644
--- a/src/allmydata/test/test_runner.py
+++ b/src/allmydata/test/test_runner.py
@@ -2,7 +2,13 @@
 from __future__ import (
     absolute_import,
 )
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
 
+from future.utils import PY2
+if PY2:
+    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 import os.path, re, sys
 from os import linesep
 

From 746e1b2664816d40936cba7d38fb5cfbae5c94ef Mon Sep 17 00:00:00 2001
From: "Jason R. Coombs" <jaraco@jaraco.com>
Date: Mon, 15 Feb 2021 12:47:26 -0500
Subject: [PATCH 19/50] Fix test failures on Python 2 by wrapping cli calls in
 unicode_to_argv.

---
 src/allmydata/test/test_runner.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/allmydata/test/test_runner.py b/src/allmydata/test/test_runner.py
index 12b658a71..d419eed37 100644
--- a/src/allmydata/test/test_runner.py
+++ b/src/allmydata/test/test_runner.py
@@ -178,7 +178,7 @@ class CreateNode(unittest.TestCase):
 
         n1 = os.path.join(basedir, command + "-n1")
         argv = ["--quiet", command, "--basedir", n1] + list(args)
-        rc, out, err = yield run_cli(*argv)
+        rc, out, err = yield run_cli(*map(unicode_to_argv, argv))
         self.failUnlessEqual(err, "")
         self.failUnlessEqual(out, "")
         self.failUnlessEqual(rc, 0)
@@ -198,7 +198,7 @@ class CreateNode(unittest.TestCase):
                 self.failUnless("\nreserved_space = 1G\n" in content)
 
         # creating the node a second time should be rejected
-        rc, out, err = yield run_cli(*argv)
+        rc, out, err = yield run_cli(*map(unicode_to_argv, argv))
         self.failIfEqual(rc, 0, str((out, err, rc)))
         self.failUnlessEqual(out, "")
         self.failUnless("is not empty." in err)
@@ -211,7 +211,7 @@ class CreateNode(unittest.TestCase):
         # test that the non --basedir form works too
         n2 = os.path.join(basedir, command + "-n2")
         argv = ["--quiet", command] + list(args) + [n2]
-        rc, out, err = yield run_cli(*argv)
+        rc, out, err = yield run_cli(*map(unicode_to_argv, argv))
         self.failUnlessEqual(err, "")
         self.failUnlessEqual(out, "")
         self.failUnlessEqual(rc, 0)
@@ -221,7 +221,7 @@ class CreateNode(unittest.TestCase):
         # test the --node-directory form
         n3 = os.path.join(basedir, command + "-n3")
         argv = ["--quiet", "--node-directory", n3, command] + list(args)
-        rc, out, err = yield run_cli(*argv)
+        rc, out, err = yield run_cli(*map(unicode_to_argv, argv))
         self.failUnlessEqual(err, "")
         self.failUnlessEqual(out, "")
         self.failUnlessEqual(rc, 0)
@@ -232,7 +232,7 @@ class CreateNode(unittest.TestCase):
             # test that the output (without --quiet) includes the base directory
             n4 = os.path.join(basedir, command + "-n4")
             argv = [command] + list(args) + [n4]
-            rc, out, err = yield run_cli(*argv)
+            rc, out, err = yield run_cli(*map(unicode_to_argv, argv))
             self.failUnlessEqual(err, "")
             self.failUnlessIn(" created in ", out)
             self.failUnlessIn(n4, out)

From 4cb8c420d321125ec9ccf0658d46dfd13128e428 Mon Sep 17 00:00:00 2001
From: "Jason R. Coombs" <jaraco@jaraco.com>
Date: Mon, 15 Feb 2021 12:49:20 -0500
Subject: [PATCH 20/50] Manually reviewed test_runner (no legacy map, filter,
 or keys found). Added docstring to tag module as fully ported.

---
 src/allmydata/test/test_runner.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/allmydata/test/test_runner.py b/src/allmydata/test/test_runner.py
index d419eed37..e7e9892fb 100644
--- a/src/allmydata/test/test_runner.py
+++ b/src/allmydata/test/test_runner.py
@@ -1,3 +1,6 @@
+"""
+Ported to Python 3
+"""
 
 from __future__ import (
     absolute_import,

From ebbe645cb613527b56d872a86ddbc19254368f0a Mon Sep 17 00:00:00 2001
From: "Jason R. Coombs" <jaraco@jaraco.com>
Date: Mon, 15 Feb 2021 13:06:42 -0500
Subject: [PATCH 21/50] Use explicit unicode literal for docstring. Now test
 passes on Python 2 when ALLOW_BYTES is indicated.

---
 src/allmydata/scripts/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/allmydata/scripts/common.py b/src/allmydata/scripts/common.py
index e50617fd7..ecb135b57 100644
--- a/src/allmydata/scripts/common.py
+++ b/src/allmydata/scripts/common.py
@@ -273,7 +273,7 @@ def get_alias(aliases, path_unicode, default):
 
 def escape_path(path):
     # type: (str) -> str
-    """
+    u"""
     Return path quoted to US-ASCII.
 
     >>> path = u'/føö/bar/☃'

From 2fb603e6032032c1c84e8df6ed3978e6d659072e Mon Sep 17 00:00:00 2001
From: "Jason R. Coombs" <jaraco@jaraco.com>
Date: Mon, 15 Feb 2021 13:10:21 -0500
Subject: [PATCH 22/50] Rewrite doctest to pass on Python 2+3.

---
 src/allmydata/scripts/common.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/allmydata/scripts/common.py b/src/allmydata/scripts/common.py
index ecb135b57..2d8d9c63a 100644
--- a/src/allmydata/scripts/common.py
+++ b/src/allmydata/scripts/common.py
@@ -277,10 +277,11 @@ def escape_path(path):
     Return path quoted to US-ASCII.
 
     >>> path = u'/føö/bar/☃'
-    >>> escape_path(path)
+    >>> escaped = escape_path(path)
+    >>> str(escaped)
     '/f%C3%B8%C3%B6/bar/%E2%98%83'
-    >>> escape_path(path).encode('ascii')
-    b'/f%C3%B8%C3%B6/bar/%E2%98%83'
+    >>> escaped.encode('ascii').decode('ascii') == escaped
+    True
     """
     # this always returns bytes, specifically US-ASCII, valid URL characters
     segments = path.split("/")

From 09f3e3f6b54e340294de84df5d599091a91564d9 Mon Sep 17 00:00:00 2001
From: "Jason R. Coombs" <jaraco@jaraco.com>
Date: Mon, 15 Feb 2021 13:26:20 -0500
Subject: [PATCH 23/50] Remove comment, superseded by docstring.

---
 src/allmydata/scripts/common.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/allmydata/scripts/common.py b/src/allmydata/scripts/common.py
index 58c7e33db..3d5237033 100644
--- a/src/allmydata/scripts/common.py
+++ b/src/allmydata/scripts/common.py
@@ -276,7 +276,7 @@ def get_alias(aliases, path_unicode, default):
 def escape_path(path):
     # type: (str) -> str
     u"""
-    Return path quoted to US-ASCII.
+    Return path quoted to US-ASCII, valid URL characters.
 
     >>> path = u'/føö/bar/☃'
     >>> escaped = escape_path(path)
@@ -285,6 +285,5 @@ def escape_path(path):
     >>> escaped.encode('ascii').decode('ascii') == escaped
     True
     """
-    # this always returns bytes, specifically US-ASCII, valid URL characters
     segments = path.split("/")
     return "/".join([urllib.parse.quote(unicode_to_url(s)) for s in segments])

From d02334bfd5ac8a1ec72943b01afd18cf33929fc2 Mon Sep 17 00:00:00 2001
From: "Jason R. Coombs" <jaraco@jaraco.com>
Date: Mon, 15 Feb 2021 14:13:59 -0500
Subject: [PATCH 24/50] Rely on futurize to expose the moved modules.

---
 src/allmydata/scripts/common.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/allmydata/scripts/common.py b/src/allmydata/scripts/common.py
index 3d5237033..c10309491 100644
--- a/src/allmydata/scripts/common.py
+++ b/src/allmydata/scripts/common.py
@@ -5,8 +5,7 @@ from __future__ import print_function
 import os, sys, textwrap
 import codecs
 from os.path import join
-
-from six.moves import urllib  # import urllib.parse
+import urllib.parse
 
 try:
     from typing import Optional

From 6118d1a2d23d181c2d3f39ef2ed1aeb0ba58742a Mon Sep 17 00:00:00 2001
From: "Jason R. Coombs" <jaraco@jaraco.com>
Date: Mon, 15 Feb 2021 14:39:27 -0500
Subject: [PATCH 25/50] Remove runner from ported modules.

---
 src/allmydata/util/_python3.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py
index 79622650c..ef846886d 100644
--- a/src/allmydata/util/_python3.py
+++ b/src/allmydata/util/_python3.py
@@ -73,7 +73,6 @@ PORTED_MODULES = [
     "allmydata.node",
     "allmydata.nodemaker",
     "allmydata.scripts.create_node",
-    "allmydata.scripts.runner",
     "allmydata.scripts.types_",
     "allmydata.stats",
     "allmydata.storage_client",

From 939988a042116f85c8d9027086633b9ff2cd0f67 Mon Sep 17 00:00:00 2001
From: "Jason R. Coombs" <jaraco@jaraco.com>
Date: Mon, 15 Feb 2021 15:13:56 -0500
Subject: [PATCH 26/50] Add workaround for compatibility on Python 2 where
 test.cli.test_create_alias expects the URL to be a byte string, broken in
 d02334bfd5ac.

---
 src/allmydata/scripts/common.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/allmydata/scripts/common.py b/src/allmydata/scripts/common.py
index c10309491..bcaa84649 100644
--- a/src/allmydata/scripts/common.py
+++ b/src/allmydata/scripts/common.py
@@ -285,4 +285,8 @@ def escape_path(path):
     True
     """
     segments = path.split("/")
-    return "/".join([urllib.parse.quote(unicode_to_url(s)) for s in segments])
+    result = "/".join([urllib.parse.quote(unicode_to_url(s)) for s in segments])
+    # fixme: test.cli.test_create_alias fails if it gets Unicode on Python 2
+    if PY2 and isinstance(result, type(u'')):
+        result = result.encode('ascii')
+    return result

From c673726139c52bfe34de7cf96f0a4b27f2ba2b2e Mon Sep 17 00:00:00 2001
From: "Jason R. Coombs" <jaraco@jaraco.com>
Date: Sat, 6 Mar 2021 14:43:44 -0500
Subject: [PATCH 27/50] Alongside unicode_to_argv, declare the argv type to
 avoid errors on Windows now that args are actually unicode.

---
 src/allmydata/test/common_util.py  | 8 ++++----
 src/allmydata/util/encodingutil.py | 9 +++++++++
 2 files changed, 13 insertions(+), 4 deletions(-)

diff --git a/src/allmydata/test/common_util.py b/src/allmydata/test/common_util.py
index f898e75b2..95ea1fc02 100644
--- a/src/allmydata/test/common_util.py
+++ b/src/allmydata/test/common_util.py
@@ -1,6 +1,6 @@
 from __future__ import print_function
 
-from future.utils import PY2, native_str, bchr, binary_type
+from future.utils import PY2, bchr, binary_type
 from future.builtins import str as future_str
 from past.builtins import unicode
 
@@ -20,7 +20,7 @@ from twisted.trial import unittest
 
 from ..util.assertutil import precondition
 from ..scripts import runner
-from allmydata.util.encodingutil import unicode_platform, get_filesystem_encoding, get_io_encoding
+from allmydata.util.encodingutil import unicode_platform, get_filesystem_encoding, get_io_encoding, argv_type
 
 
 def skip_if_cannot_represent_filename(u):
@@ -75,8 +75,8 @@ def run_cli_native(verb, *args, **kwargs):
     nodeargs = kwargs.pop("nodeargs", [])
     encoding = kwargs.pop("encoding", None)
     precondition(
-        all(isinstance(arg, native_str) for arg in [verb] + nodeargs + list(args)),
-        "arguments to run_cli must be a native string -- convert using unicode_to_argv",
+        all(isinstance(arg, argv_type) for arg in [verb] + nodeargs + list(args)),
+        "arguments to run_cli must be {argv_type} -- convert using unicode_to_argv".format(argv_type=argv_type),
         verb=verb,
         args=args,
         nodeargs=nodeargs,
diff --git a/src/allmydata/util/encodingutil.py b/src/allmydata/util/encodingutil.py
index 483871b5d..69e858e19 100644
--- a/src/allmydata/util/encodingutil.py
+++ b/src/allmydata/util/encodingutil.py
@@ -13,6 +13,7 @@ from __future__ import print_function
 from __future__ import unicode_literals
 
 from future.utils import PY2, PY3, native_str
+from future.builtins import str as future_str
 if PY2:
     # We omit str() because that seems too tricky to get right.
     from builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, max, min  # noqa: F401
@@ -142,6 +143,14 @@ def unicode_to_argv(s, mangle=False):
     return ensure_str(s)
 
 
+# According to unicode_to_argv above, the expected type for
+# cli args depends on the platform, so capture that expectation.
+argv_type = future_str if sys.platform == "win32" else native_str
+"""
+The expected type for args to a subprocess
+"""
+
+
 def unicode_to_url(s):
     """
     Encode an unicode object used in an URL to bytes.

From ee99c610b3a4bd57ae93ae8c557e4dca548635e4 Mon Sep 17 00:00:00 2001
From: "Jason R. Coombs" <jaraco@jaraco.com>
Date: Sat, 6 Mar 2021 16:06:24 -0500
Subject: [PATCH 28/50] argv_type on Windows can be either

---
 src/allmydata/util/encodingutil.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/allmydata/util/encodingutil.py b/src/allmydata/util/encodingutil.py
index 69e858e19..637374064 100644
--- a/src/allmydata/util/encodingutil.py
+++ b/src/allmydata/util/encodingutil.py
@@ -145,7 +145,7 @@ def unicode_to_argv(s, mangle=False):
 
 # According to unicode_to_argv above, the expected type for
 # cli args depends on the platform, so capture that expectation.
-argv_type = future_str if sys.platform == "win32" else native_str
+argv_type = (future_str, native_str) if sys.platform == "win32" else native_str
 """
 The expected type for args to a subprocess
 """

From 7c2a0685bb7ac5dea260ecec961f9ca47d010822 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Fri, 12 Mar 2021 11:22:19 -0500
Subject: [PATCH 29/50] Port test_create.py to Python 3.

---
 src/allmydata/test/cli/test_create.py | 12 ++++++++++++
 src/allmydata/test/common_util.py     | 12 +++++++++++-
 src/allmydata/util/_python3.py        |  2 ++
 3 files changed, 25 insertions(+), 1 deletion(-)

diff --git a/src/allmydata/test/cli/test_create.py b/src/allmydata/test/cli/test_create.py
index aee07a671..282f26163 100644
--- a/src/allmydata/test/cli/test_create.py
+++ b/src/allmydata/test/cli/test_create.py
@@ -1,3 +1,15 @@
+"""
+Ported to Python 3.
+"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from future.utils import PY2
+if PY2:
+    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
+
 import os
 import mock
 from twisted.trial import unittest
diff --git a/src/allmydata/test/common_util.py b/src/allmydata/test/common_util.py
index 95ea1fc02..f62cd34cc 100644
--- a/src/allmydata/test/common_util.py
+++ b/src/allmydata/test/common_util.py
@@ -20,7 +20,7 @@ from twisted.trial import unittest
 
 from ..util.assertutil import precondition
 from ..scripts import runner
-from allmydata.util.encodingutil import unicode_platform, get_filesystem_encoding, get_io_encoding, argv_type
+from allmydata.util.encodingutil import unicode_platform, get_filesystem_encoding, get_io_encoding, argv_type, unicode_to_argv
 
 
 def skip_if_cannot_represent_filename(u):
@@ -49,6 +49,13 @@ def _getvalue(io):
     return io.read()
 
 
+def maybe_unicode_to_argv(o):
+    """Convert object to argv form if necessary."""
+    if isinstance(o, unicode):
+        return unicode_to_argv(o)
+    return o
+
+
 def run_cli_native(verb, *args, **kwargs):
     """
     Run a Tahoe-LAFS CLI command specified as bytes (on Python 2) or Unicode
@@ -74,6 +81,9 @@ def run_cli_native(verb, *args, **kwargs):
     """
     nodeargs = kwargs.pop("nodeargs", [])
     encoding = kwargs.pop("encoding", None)
+    verb = maybe_unicode_to_argv(verb)
+    args = [maybe_unicode_to_argv(a) for a in args]
+    nodeargs = [maybe_unicode_to_argv(a) for a in nodeargs]
     precondition(
         all(isinstance(arg, argv_type) for arg in [verb] + nodeargs + list(args)),
         "arguments to run_cli must be {argv_type} -- convert using unicode_to_argv".format(argv_type=argv_type),
diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py
index f020ee8d7..5bcf64f0f 100644
--- a/src/allmydata/util/_python3.py
+++ b/src/allmydata/util/_python3.py
@@ -138,6 +138,8 @@ PORTED_MODULES = [
 ]
 
 PORTED_TEST_MODULES = [
+    "allmydata.test.cli.test_create",
+    
     "allmydata.test.mutable.test_checker",
     "allmydata.test.mutable.test_datahandle",
     "allmydata.test.mutable.test_different_encoding",

From 68d342ee298ccb24f9698218871e6370f41c4963 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Fri, 12 Mar 2021 11:26:14 -0500
Subject: [PATCH 30/50] Get rid of trailing whitespace.

---
 src/allmydata/util/_python3.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py
index 5bcf64f0f..51dd8111d 100644
--- a/src/allmydata/util/_python3.py
+++ b/src/allmydata/util/_python3.py
@@ -139,7 +139,7 @@ PORTED_MODULES = [
 
 PORTED_TEST_MODULES = [
     "allmydata.test.cli.test_create",
-    
+
     "allmydata.test.mutable.test_checker",
     "allmydata.test.mutable.test_datahandle",
     "allmydata.test.mutable.test_different_encoding",

From 4b6f84b821979fde6139cd397fb9562e5e5b88f2 Mon Sep 17 00:00:00 2001
From: Chad Whitacre <chad@zetaweb.com>
Date: Sun, 28 Feb 2021 12:19:47 -0500
Subject: [PATCH 31/50] Port testing to Python 3

---
 newsfragments/3621.minor     |  0
 src/allmydata/testing/web.py | 14 ++++++++++++--
 2 files changed, 12 insertions(+), 2 deletions(-)
 create mode 100644 newsfragments/3621.minor

diff --git a/newsfragments/3621.minor b/newsfragments/3621.minor
new file mode 100644
index 000000000..e69de29bb
diff --git a/src/allmydata/testing/web.py b/src/allmydata/testing/web.py
index 12f5ef40c..0c3389956 100644
--- a/src/allmydata/testing/web.py
+++ b/src/allmydata/testing/web.py
@@ -6,10 +6,20 @@
 # This file is part of Tahoe-LAFS.
 #
 # See the docs/about.rst file for licensing information.
+"""Test-helpers for clients that use the WebUI.
+
+Ported to Python 3.
 
 """
-Test-helpers for clients that use the WebUI.
-"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from future.utils import PY2
+if PY2:
+    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
+
 
 import hashlib
 

From bde424c7f985494b35b65e38869f651794b109f3 Mon Sep 17 00:00:00 2001
From: Chad Whitacre <chad@zetaweb.com>
Date: Sat, 13 Mar 2021 09:00:01 -0500
Subject: [PATCH 32/50] Finish porting testing & test_testing

---
 src/allmydata/test/test_testing.py | 21 ++++++++++++++++-----
 src/allmydata/testing/web.py       | 26 ++++++++++++++++----------
 src/allmydata/util/_python3.py     |  2 ++
 3 files changed, 34 insertions(+), 15 deletions(-)

diff --git a/src/allmydata/test/test_testing.py b/src/allmydata/test/test_testing.py
index ec6935914..527b235bd 100644
--- a/src/allmydata/test/test_testing.py
+++ b/src/allmydata/test/test_testing.py
@@ -9,7 +9,18 @@
 
 """
 Tests for the allmydata.testing helpers
+
+Ported to Python 3.
+
 """
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from future.utils import PY2
+if PY2:
+    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
 from twisted.internet.defer import (
     inlineCallbacks,
@@ -79,7 +90,7 @@ class FakeWebTest(TestCase):
             self.assertThat(cap, IsInstance(CHKFileURI))
 
             resp = yield http_client.get(
-                "http://example.com/uri?uri={}".format(cap.to_string())
+                b"http://example.com/uri?uri=" + cap.to_string()
             )
             self.assertThat(resp.code, Equals(200))
 
@@ -88,7 +99,7 @@ class FakeWebTest(TestCase):
             # using the form "/uri/<cap>" is also valid
 
             resp = yield http_client.get(
-                "http://example.com/uri/{}".format(cap.to_string())
+                b"http://example.com/uri?uri=" + cap.to_string()
             )
             self.assertEqual(resp.code, 200)
 
@@ -136,9 +147,9 @@ class FakeWebTest(TestCase):
         """
 
         http_client = create_tahoe_treq_client()
-        cap_gen = capability_generator("URI:CHK:")
-
-        uri = DecodedURL.from_text(u"http://example.com/uri?uri={}".format(next(cap_gen)))
+        cap_gen = capability_generator(b"URI:CHK:")
+        cap = next(cap_gen).decode('ascii')
+        uri = DecodedURL.from_text(u"http://example.com/uri?uri={}".format(cap))
         resp = http_client.get(uri.to_uri().to_text())
 
         self.assertThat(
diff --git a/src/allmydata/testing/web.py b/src/allmydata/testing/web.py
index 0c3389956..34f5087e8 100644
--- a/src/allmydata/testing/web.py
+++ b/src/allmydata/testing/web.py
@@ -94,16 +94,19 @@ def capability_generator(kind):
     given kind. The N, K and size values aren't related to anything
     real.
 
-    :param str kind: the kind of capability, like `URI:CHK`
+    :param bytes kind: the kind of capability, like `URI:CHK`
 
     :returns: a generator that yields new capablities of a particular
         kind.
     """
+    if not isinstance(kind, bytes):
+        raise TypeError("'kind' must be bytes")
+
     if kind not in KNOWN_CAPABILITIES:
         raise ValueError(
             "Unknown capability kind '{} (valid are {})'".format(
                 kind,
-                ", ".join(KNOWN_CAPABILITIES),
+                b", ".join(KNOWN_CAPABILITIES),
             )
         )
     # what we do here is to start with empty hashers for the key and
@@ -118,16 +121,16 @@ def capability_generator(kind):
     # capabilities are "prefix:<128-bits-base32>:<256-bits-base32>:N:K:size"
     while True:
         number += 1
-        key_hasher.update("\x00")
-        ueb_hasher.update("\x00")
+        key_hasher.update(b"\x00")
+        ueb_hasher.update(b"\x00")
 
         key = base32.b2a(key_hasher.digest()[:16])  # key is 16 bytes
         ueb_hash = base32.b2a(ueb_hasher.digest())  # ueb hash is 32 bytes
 
         cap = u"{kind}{key}:{ueb_hash}:{n}:{k}:{size}".format(
-            kind=kind,
-            key=key,
-            ueb_hash=ueb_hash,
+            kind=kind.decode('ascii'),
+            key=key.decode('ascii'),
+            ueb_hash=ueb_hash.decode('ascii'),
             n=1,
             k=1,
             size=number * 1000,
@@ -164,6 +167,8 @@ class _FakeTahoeUriHandler(Resource, object):
 
         :returns: a two-tuple: a bool (True if the data is freshly added) and a capability-string
         """
+        if not isinstance(kind, bytes):
+            raise TypeError("'kind' must be bytes")
         if not isinstance(data, bytes):
             raise TypeError("'data' must be bytes")
 
@@ -181,7 +186,7 @@ class _FakeTahoeUriHandler(Resource, object):
 
     def render_PUT(self, request):
         data = request.content.read()
-        fresh, cap = self.add_data("URI:CHK:", data)
+        fresh, cap = self.add_data(b"URI:CHK:", data)
         if fresh:
             request.setResponseCode(http.CREATED)  # real code does this for brand-new files
         else:
@@ -193,7 +198,7 @@ class _FakeTahoeUriHandler(Resource, object):
         data = request.content.read()
 
         type_to_kind = {
-            "mkdir-immutable": "URI:DIR2-CHK:"
+            "mkdir-immutable": b"URI:DIR2-CHK:"
         }
         kind = type_to_kind[t]
         fresh, cap = self.add_data(kind, data)
@@ -216,9 +221,10 @@ class _FakeTahoeUriHandler(Resource, object):
 
         # the user gave us a capability; if our Grid doesn't have any
         # data for it, that's an error.
+        capability = capability.encode('ascii')
         if capability not in self.data:
             request.setResponseCode(http.BAD_REQUEST)
-            return u"No data for '{}'".format(capability).decode("ascii")
+            return u"No data for '{}'".format(capability.decode('ascii'))
 
         return self.data[capability]
 
diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py
index e5e616674..358990d94 100644
--- a/src/allmydata/util/_python3.py
+++ b/src/allmydata/util/_python3.py
@@ -87,6 +87,7 @@ PORTED_MODULES = [
     "allmydata.storage.shares",
     "allmydata.test.no_network",
     "allmydata.test.mutable.util",
+    "allmydata.testing",
     "allmydata.unknown",
     "allmydata.uri",
     "allmydata.util._python3",
@@ -205,6 +206,7 @@ PORTED_TEST_MODULES = [
     # should be done once CLI is ported.
     "allmydata.test.test_system",
 
+    "allmydata.test.test_testing",
     "allmydata.test.test_time_format",
     "allmydata.test.test_upload",
     "allmydata.test.test_uri",

From 0056809a5512efdc6cea1363bb04e24c8fd3696a Mon Sep 17 00:00:00 2001
From: Chad Whitacre <chad@zetaweb.com>
Date: Sun, 14 Mar 2021 17:34:24 -0400
Subject: [PATCH 33/50] Clean up error message (for mypy)

---
 src/allmydata/testing/web.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/allmydata/testing/web.py b/src/allmydata/testing/web.py
index 34f5087e8..8bd21f7d0 100644
--- a/src/allmydata/testing/web.py
+++ b/src/allmydata/testing/web.py
@@ -105,8 +105,8 @@ def capability_generator(kind):
     if kind not in KNOWN_CAPABILITIES:
         raise ValueError(
             "Unknown capability kind '{} (valid are {})'".format(
-                kind,
-                b", ".join(KNOWN_CAPABILITIES),
+                kind.decode('ascii'),
+                ", ".join([x.decode('ascii') for x in KNOWN_CAPABILITIES]),
             )
         )
     # what we do here is to start with empty hashers for the key and

From 6752e68d40cabe95435b3eca6a167aa352cd00c8 Mon Sep 17 00:00:00 2001
From: Chad Whitacre <chad@zetaweb.com>
Date: Mon, 15 Mar 2021 06:15:47 -0400
Subject: [PATCH 34/50] Review files, pick off a typo

---
 src/allmydata/testing/web.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/allmydata/testing/web.py b/src/allmydata/testing/web.py
index 8bd21f7d0..bb858b555 100644
--- a/src/allmydata/testing/web.py
+++ b/src/allmydata/testing/web.py
@@ -104,7 +104,7 @@ def capability_generator(kind):
 
     if kind not in KNOWN_CAPABILITIES:
         raise ValueError(
-            "Unknown capability kind '{} (valid are {})'".format(
+            "Unknown capability kind '{}' (valid are {})".format(
                 kind.decode('ascii'),
                 ", ".join([x.decode('ascii') for x in KNOWN_CAPABILITIES]),
             )

From dee9f622a96dad6f3f4f74e05d31849c191f524d Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Mon, 15 Mar 2021 10:50:00 -0400
Subject: [PATCH 35/50] Tests pass on Python 3.

---
 src/allmydata/frontends/auth.py | 15 ++++++++-------
 src/allmydata/test/test_auth.py |  2 +-
 2 files changed, 9 insertions(+), 8 deletions(-)

diff --git a/src/allmydata/frontends/auth.py b/src/allmydata/frontends/auth.py
index de406d604..7f81572fe 100644
--- a/src/allmydata/frontends/auth.py
+++ b/src/allmydata/frontends/auth.py
@@ -7,6 +7,7 @@ from twisted.cred import error, checkers, credentials
 from twisted.conch.ssh import keys
 from twisted.conch.checkers import SSHPublicKeyChecker, InMemorySSHKeyDB
 
+from allmydata.util.dictutil import BytesKeyDict
 from allmydata.util import base32
 from allmydata.util.fileutil import abspath_expanduser_unicode
 
@@ -28,18 +29,18 @@ class AccountFileChecker(object):
                             credentials.ISSHPrivateKey)
     def __init__(self, client, accountfile):
         self.client = client
-        self.passwords = {}
-        pubkeys = {}
-        self.rootcaps = {}
-        with open(abspath_expanduser_unicode(accountfile), "r") as f:
+        self.passwords = BytesKeyDict()
+        pubkeys = BytesKeyDict()
+        self.rootcaps = BytesKeyDict()
+        with open(abspath_expanduser_unicode(accountfile), "rb") as f:
             for line in f:
                 line = line.strip()
-                if line.startswith("#") or not line:
+                if line.startswith(b"#") or not line:
                     continue
                 name, passwd, rest = line.split(None, 2)
-                if passwd.startswith("ssh-"):
+                if passwd.startswith(b"ssh-"):
                     bits = rest.split()
-                    keystring = " ".join([passwd] + bits[:-1])
+                    keystring = b" ".join([passwd] + bits[:-1])
                     key = keys.Key.fromString(keystring)
                     rootcap = bits[-1]
                     pubkeys[name] = [key]
diff --git a/src/allmydata/test/test_auth.py b/src/allmydata/test/test_auth.py
index 5670933df..1f3db687a 100644
--- a/src/allmydata/test/test_auth.py
+++ b/src/allmydata/test/test_auth.py
@@ -35,7 +35,7 @@ DUMMY_ACCOUNTS = u"""\
 alice password URI:DIR2:aaaaaaaaaaaaaaaaaaaaaaaaaa:1111111111111111111111111111111111111111111111111111
 bob sekrit URI:DIR2:bbbbbbbbbbbbbbbbbbbbbbbbbb:2222222222222222222222222222222222222222222222222222
 carol {key} URI:DIR2:cccccccccccccccccccccccccc:3333333333333333333333333333333333333333333333333333
-""".format(key=DUMMY_KEY.public().toString("openssh")).encode("ascii")
+""".format(key=str(DUMMY_KEY.public().toString("openssh"), "ascii")).encode("ascii")
 
 class AccountFileCheckerKeyTests(unittest.TestCase):
     """

From 01ee67fa43db64a734fd3da6eb0296c3085da4f6 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Mon, 15 Mar 2021 11:03:39 -0400
Subject: [PATCH 36/50] Port to Python 3.

---
 src/allmydata/test/test_auth.py | 9 ++++++++-
 src/allmydata/util/_python3.py  | 1 +
 2 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/src/allmydata/test/test_auth.py b/src/allmydata/test/test_auth.py
index 1f3db687a..f808f72ab 100644
--- a/src/allmydata/test/test_auth.py
+++ b/src/allmydata/test/test_auth.py
@@ -1,4 +1,11 @@
-# Python 2 compatibility
+"""
+Ported to Python 3.
+"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
 from future.utils import PY2
 if PY2:
     from future.builtins import str  # noqa: F401
diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py
index e5e616674..0ac3f76e4 100644
--- a/src/allmydata/util/_python3.py
+++ b/src/allmydata/util/_python3.py
@@ -152,6 +152,7 @@ PORTED_TEST_MODULES = [
     "allmydata.test.mutable.test_update",
     "allmydata.test.mutable.test_version",
     "allmydata.test.test_abbreviate",
+    "allmydata.test.test_auth",
     "allmydata.test.test_base32",
     "allmydata.test.test_base62",
     "allmydata.test.test_checker",

From 6bce1e8459652d95d1f3bfb29c32022fe229472b Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Mon, 15 Mar 2021 11:03:59 -0400
Subject: [PATCH 37/50] News file.

---
 newsfragments/3635.minor | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 create mode 100644 newsfragments/3635.minor

diff --git a/newsfragments/3635.minor b/newsfragments/3635.minor
new file mode 100644
index 000000000..e69de29bb

From 47f2c143d0ab57f84d2800a24c8176e92e9479b7 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Mon, 15 Mar 2021 11:07:03 -0400
Subject: [PATCH 38/50] Tests pass on Python 3.

---
 src/allmydata/test/test_hung_server.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/allmydata/test/test_hung_server.py b/src/allmydata/test/test_hung_server.py
index d3a42d416..8c32a1663 100644
--- a/src/allmydata/test/test_hung_server.py
+++ b/src/allmydata/test/test_hung_server.py
@@ -14,8 +14,8 @@ from allmydata.test.common import ShouldFailMixin
 from allmydata.util.pollmixin import PollMixin
 from allmydata.interfaces import NotEnoughSharesError
 
-immutable_plaintext = "data" * 10000
-mutable_plaintext = "muta" * 10000
+immutable_plaintext = b"data" * 10000
+mutable_plaintext = b"muta" * 10000
 
 class HungServerDownloadTest(GridTestMixin, ShouldFailMixin, PollMixin,
                              unittest.TestCase):
@@ -105,7 +105,7 @@ class HungServerDownloadTest(GridTestMixin, ShouldFailMixin, PollMixin,
                 self.shares = self.find_uri_shares(self.uri)
             d.addCallback(_uploaded_mutable)
         else:
-            data = upload.Data(immutable_plaintext, convergence="")
+            data = upload.Data(immutable_plaintext, convergence=b"")
             d = self.c0.upload(data)
             def _uploaded_immutable(upload_res):
                 self.uri = upload_res.get_uri()
@@ -262,7 +262,7 @@ class HungServerDownloadTest(GridTestMixin, ShouldFailMixin, PollMixin,
             # is shut off. That will leave 4 OVERDUE and 1
             # stuck-but-not-overdue, for a total of 5 requests in in
             # _sf.pending_requests
-            for t in self._sf.overdue_timers.values()[:4]:
+            for t in list(self._sf.overdue_timers.values())[:4]:
                 t.reset(-1.0)
             # the timers ought to fire before the eventual-send does
             return fireEventually()

From 706b77aaea0b01dfa4b22d9504b0c8607a3f1ac3 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Mon, 15 Mar 2021 11:08:09 -0400
Subject: [PATCH 39/50] Port to Python 3.

---
 src/allmydata/test/test_hung_server.py | 12 ++++++++++++
 src/allmydata/util/_python3.py         |  1 +
 2 files changed, 13 insertions(+)

diff --git a/src/allmydata/test/test_hung_server.py b/src/allmydata/test/test_hung_server.py
index 8c32a1663..490315500 100644
--- a/src/allmydata/test/test_hung_server.py
+++ b/src/allmydata/test/test_hung_server.py
@@ -1,5 +1,17 @@
 # -*- coding: utf-8 -*-
 
+"""
+Ported to Python 3.
+"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from future.utils import PY2
+if PY2:
+    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
+
 import os, shutil
 from twisted.trial import unittest
 from twisted.internet import defer
diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py
index 0ac3f76e4..a43c70e18 100644
--- a/src/allmydata/util/_python3.py
+++ b/src/allmydata/util/_python3.py
@@ -180,6 +180,7 @@ PORTED_TEST_MODULES = [
     "allmydata.test.test_hashutil",
     "allmydata.test.test_helper",
     "allmydata.test.test_humanreadable",
+    "allmydata.test.test_hung_server",
     "allmydata.test.test_immutable",
     "allmydata.test.test_introducer",
     "allmydata.test.test_iputil",

From dc057b6b0a983938317866a3dae0020fd0133202 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Tue, 16 Mar 2021 14:48:13 -0400
Subject: [PATCH 40/50] news fragment

---
 newsfragments/3629.feature | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 newsfragments/3629.feature

diff --git a/newsfragments/3629.feature b/newsfragments/3629.feature
new file mode 100644
index 000000000..cdca48a18
--- /dev/null
+++ b/newsfragments/3629.feature
@@ -0,0 +1 @@
+The NixOS-packaged Tahoe-LAFS now knows its own version.

From f261b7b2971f2aaf93df03f5d0b2373c36cfba57 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Tue, 16 Mar 2021 15:00:19 -0400
Subject: [PATCH 41/50] Apply our own filtering logic

Notably, leave .git in so that Tahoe's built-in auto-versioning thing will
work.
---
 nix/tahoe-lafs.nix | 43 ++++++++++++++++++++++++++++++++++++-------
 1 file changed, 36 insertions(+), 7 deletions(-)

diff --git a/nix/tahoe-lafs.nix b/nix/tahoe-lafs.nix
index c831b6b7b..57d718e01 100644
--- a/nix/tahoe-lafs.nix
+++ b/nix/tahoe-lafs.nix
@@ -1,5 +1,5 @@
 { fetchFromGitHub, lib
-, python
+, git, python
 , twisted, foolscap, zfec
 , setuptools, setuptoolsTrial, pyasn1, zope_interface
 , service-identity, pyyaml, magic-wormhole, treq, appdirs
@@ -9,7 +9,35 @@
 python.pkgs.buildPythonPackage rec {
   version = "1.14.0.dev";
   name = "tahoe-lafs-${version}";
-  src = lib.cleanSource ../.;
+  src = lib.cleanSourceWith {
+    src = ../.;
+    filter = name: type:
+      let
+        basename = baseNameOf name;
+
+        split = lib.splitString ".";
+        join = builtins.concatStringsSep ".";
+        ext = join (builtins.tail (split basename));
+
+        # Build up a bunch of knowledge about what kind of file this is.
+        isTox = type == "directory" && basename == ".tox";
+        isTrialTemp = type == "directory" && basename == "_trial_temp";
+        isVersion = basename == "version.py";
+        isBytecode = ext == "pyc" || ext == "pyo";
+        isBackup = lib.hasSuffix "~" basename;
+        isTemporary = lib.hasPrefix "#" basename && lib.hasSuffix "#" basename;
+        isSymlink = type == "symlink";
+      in
+      # Exclude all these things
+      ! (isTrialTemp
+      || isTox
+      || isVersion
+      || isBytecode
+      || isBackup
+      || isTemporary
+      || isSymlink
+      );
+  };
 
   postPatch = ''
     # Chroots don't have /etc/hosts and /etc/resolv.conf, so work around
@@ -24,13 +52,10 @@ python.pkgs.buildPythonPackage rec {
     # tests with in a module.
 
     # Many of these tests don't properly skip when i2p or tor dependencies are
-    # not supplied (and we are not supplying them). test_client.py fails because
-    # version is "unknown" on Nix.
-    # see https://tahoe-lafs.org/trac/tahoe-lafs/ticket/3629 for the latter.
+    # not supplied (and we are not supplying them).
     rm src/allmydata/test/test_i2p_provider.py
     rm src/allmydata/test/test_connections.py
     rm src/allmydata/test/cli/test_create.py
-    rm src/allmydata/test/test_client.py
 
     # Since we're deleting files, this complains they're missing. For now Nix
     # is Python 2-only, anyway, so these tests don't add anything yet.
@@ -38,6 +63,10 @@ python.pkgs.buildPythonPackage rec {
   '';
 
 
+  nativeBuildInputs = [
+    git
+  ];
+
   propagatedBuildInputs = with python.pkgs; [
     twisted foolscap zfec appdirs
     setuptoolsTrial pyasn1 zope_interface
@@ -56,6 +85,6 @@ python.pkgs.buildPythonPackage rec {
   ];
 
   checkPhase = ''
-    ${python}/bin/python -m twisted.trial -j $NIX_BUILD_CORES allmydata
+    ${python}/bin/python -m twisted.trial -j $NIX_BUILD_CORES allmydata.test.test_client
   '';
 }

From cebd0f5547171252713c693b22ebe967fbcb6db8 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Tue, 16 Mar 2021 15:06:51 -0400
Subject: [PATCH 42/50] Switch back to the full test suite

---
 nix/tahoe-lafs.nix | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/nix/tahoe-lafs.nix b/nix/tahoe-lafs.nix
index 57d718e01..6c3c68343 100644
--- a/nix/tahoe-lafs.nix
+++ b/nix/tahoe-lafs.nix
@@ -85,6 +85,6 @@ python.pkgs.buildPythonPackage rec {
   ];
 
   checkPhase = ''
-    ${python}/bin/python -m twisted.trial -j $NIX_BUILD_CORES allmydata.test.test_client
+    ${python}/bin/python -m twisted.trial -j $NIX_BUILD_CORES allmydata
   '';
 }

From aa9216685940980874ebd29512e17fdbdce18383 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Wed, 17 Mar 2021 14:33:25 -0400
Subject: [PATCH 43/50] news fragment

---
 newsfragments/3637.minor | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 create mode 100644 newsfragments/3637.minor

diff --git a/newsfragments/3637.minor b/newsfragments/3637.minor
new file mode 100644
index 000000000..e69de29bb

From 4c15db2019edcb7054fbbb03cec8666f9e156ac3 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Wed, 17 Mar 2021 14:33:32 -0400
Subject: [PATCH 44/50] ignore this impossible-to-fix error

also link to a twisted bug report for making it possible
---
 src/allmydata/test/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/allmydata/test/common.py b/src/allmydata/test/common.py
index 533cc4fc6..ae1616fe4 100644
--- a/src/allmydata/test/common.py
+++ b/src/allmydata/test/common.py
@@ -323,7 +323,7 @@ class AdoptedServerPort(object):
     """
     prefix = "adopt-socket"
 
-    def parseStreamServer(self, reactor, fd):
+    def parseStreamServer(self, reactor, fd): # type: ignore # https://twistedmatrix.com/trac/ticket/10134
         log.msg("Adopting {}".format(fd))
         # AdoptedStreamServerEndpoint wants to own the file descriptor.  It
         # will duplicate it and then close the one we pass in.  This means it

From ac6b2c61773386cd878e7ccde1172b170a56900c Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Wed, 17 Mar 2021 14:33:58 -0400
Subject: [PATCH 45/50] add missing producer method to this dummy

---
 src/allmydata/test/common.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/allmydata/test/common.py b/src/allmydata/test/common.py
index ae1616fe4..94a59f5c9 100644
--- a/src/allmydata/test/common.py
+++ b/src/allmydata/test/common.py
@@ -425,6 +425,9 @@ class DummyProducer(object):
     def resumeProducing(self):
         pass
 
+    def stopProducing(self):
+        pass
+
 @implementer(IImmutableFileNode)
 class FakeCHKFileNode(object):  # type: ignore # incomplete implementation
     """I provide IImmutableFileNode, but all of my data is stored in a

From 9425197832eeb3cd9e8e51e6fa9eeda2110b6ec3 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Wed, 17 Mar 2021 14:34:09 -0400
Subject: [PATCH 46/50] twisted.trial.unittest has no main, oops

---
 src/allmydata/test/test_multi_introducers.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/src/allmydata/test/test_multi_introducers.py b/src/allmydata/test/test_multi_introducers.py
index 520a5a69a..bb22d551f 100644
--- a/src/allmydata/test/test_multi_introducers.py
+++ b/src/allmydata/test/test_multi_introducers.py
@@ -175,6 +175,3 @@ class NoDefault(unittest.TestCase):
         self.yaml_path.setContent(yamlutil.safe_dump(connections))
         myclient = yield create_client(self.basedir)
         self.assertEquals(len(myclient.introducer_clients), 0)
-
-if __name__ == "__main__":
-    unittest.main()

From 9ae3f2074f1e4f0dd79d839d6c4dd5f87b4ab249 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Wed, 17 Mar 2021 14:34:19 -0400
Subject: [PATCH 47/50] use the inherited implementation!

---
 src/allmydata/test/test_repairer.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/src/allmydata/test/test_repairer.py b/src/allmydata/test/test_repairer.py
index 63a54a505..88696000c 100644
--- a/src/allmydata/test/test_repairer.py
+++ b/src/allmydata/test/test_repairer.py
@@ -30,9 +30,6 @@ MAX_DELTA_READS = 10 * READ_LEEWAY # N = 10
 timeout=240 # François's ARM box timed out after 120 seconds of Verifier.test_corrupt_crypttext_hashtree
 
 class RepairTestMixin(object):
-    def failUnlessIsInstance(self, x, xtype):
-        self.failUnless(isinstance(x, xtype), x)
-
     def _count_reads(self):
         sum_of_read_counts = 0
         for (i, ss, storedir) in self.iterate_servers():

From 950a03cf644911dd85d9174b6463186e3728826c Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Wed, 17 Mar 2021 14:34:32 -0400
Subject: [PATCH 48/50] ignore two examples of this error

and link to twisted bug report about it
---
 src/allmydata/test/web/test_web.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/allmydata/test/web/test_web.py b/src/allmydata/test/web/test_web.py
index 803aba34e..6b25305c6 100644
--- a/src/allmydata/test/web/test_web.py
+++ b/src/allmydata/test/web/test_web.py
@@ -108,7 +108,7 @@ class FakeNodeMaker(NodeMaker):
         return n.create(contents, version=version)
 
 class FakeUploader(service.Service):
-    name = "uploader"
+    name = "uploader" # type: ignore # https://twistedmatrix.com/trac/ticket/10135
     helper_furl = None
     helper_connected = False
 
@@ -254,7 +254,7 @@ class FakeLeaseChecker(object):
                 "remaining-wait-time": 0}
 
 class FakeStorageServer(service.MultiService):
-    name = 'storage'
+    name = 'storage' # type: ignore # https://twistedmatrix.com/trac/ticket/10135
     def __init__(self, nodeid, nickname):
         service.MultiService.__init__(self)
         self.my_nodeid = nodeid

From ca125193cfbc0eb22a994cf12aa474fa147dc0df Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Wed, 17 Mar 2021 15:08:21 -0400
Subject: [PATCH 49/50] Fix module name.

---
 src/allmydata/util/_python3.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py
index f6afc77d0..961cc7e15 100644
--- a/src/allmydata/util/_python3.py
+++ b/src/allmydata/util/_python3.py
@@ -89,7 +89,7 @@ PORTED_MODULES = [
     "allmydata.test.no_network",
     "allmydata.test.matchers",
     "allmydata.test.mutable.util",
-    "allmydata.testing",
+    "allmydata.testing.web",
     "allmydata.unknown",
     "allmydata.uri",
     "allmydata.util._python3",

From 2eba96de61e89f3b836a38ff288790893ba21e30 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Wed, 17 Mar 2021 15:27:45 -0400
Subject: [PATCH 50/50] Address review comments.

---
 src/allmydata/scripts/common.py   | 5 ++---
 src/allmydata/scripts/runner.py   | 5 +++--
 src/allmydata/test/test_runner.py | 9 ++++-----
 3 files changed, 9 insertions(+), 10 deletions(-)

diff --git a/src/allmydata/scripts/common.py b/src/allmydata/scripts/common.py
index bcaa84649..36330f535 100644
--- a/src/allmydata/scripts/common.py
+++ b/src/allmydata/scripts/common.py
@@ -1,6 +1,7 @@
 # coding: utf-8
 
 from __future__ import print_function
+from six import ensure_str
 
 import os, sys, textwrap
 import codecs
@@ -286,7 +287,5 @@ def escape_path(path):
     """
     segments = path.split("/")
     result = "/".join([urllib.parse.quote(unicode_to_url(s)) for s in segments])
-    # fixme: test.cli.test_create_alias fails if it gets Unicode on Python 2
-    if PY2 and isinstance(result, type(u'')):
-        result = result.encode('ascii')
+    result = ensure_str(result, "ascii")
     return result
diff --git a/src/allmydata/scripts/runner.py b/src/allmydata/scripts/runner.py
index 705fa37bf..ada3c2dfc 100644
--- a/src/allmydata/scripts/runner.py
+++ b/src/allmydata/scripts/runner.py
@@ -174,8 +174,9 @@ def _maybe_enable_eliot_logging(options, reactor):
     return options
 
 def run():
-    if not six.PY2:
-        warnings.warn("Support for Python 3 is experimental. Use at your own risk.")
+    if six.PY3:
+        warnings.warn("Support for Python 3 is an incomplete work-in-progress."
+                      " Use at your own risk.")
 
     if sys.platform == "win32":
         from allmydata.windows.fixups import initialize
diff --git a/src/allmydata/test/test_runner.py b/src/allmydata/test/test_runner.py
index e7e9892fb..f6a7c2ee1 100644
--- a/src/allmydata/test/test_runner.py
+++ b/src/allmydata/test/test_runner.py
@@ -12,6 +12,9 @@ from __future__ import unicode_literals
 from future.utils import PY2
 if PY2:
     from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
+
+from six import ensure_text
+
 import os.path, re, sys
 from os import linesep
 
@@ -19,8 +22,6 @@ from eliot import (
     log_call,
 )
 
-import six
-
 from twisted.trial import unittest
 
 from twisted.internet import reactor
@@ -83,9 +84,7 @@ def run_bintahoe(extra_argv, python_options=None):
     :return: A three-tuple of stdout (unicode), stderr (unicode), and the
         child process "returncode" (int).
     """
-    # fixme: below, 'unicode_to_argv' is called so ensure that
-    # executable is unicode to support that expectation.
-    executable = sys.executable.decode('utf-8') if six.PY2 else sys.executable
+    executable = ensure_text(sys.executable)
     argv = [executable]
     if python_options is not None:
         argv.extend(python_options)