From ff182e69c1bc699503f38477c75197671f0e67ae Mon Sep 17 00:00:00 2001
From: meejah <meejah@meejah.ca>
Date: Sun, 20 Dec 2020 19:10:00 -0700
Subject: [PATCH 01/37] signatures are detached

---
 docs/release-checklist.rst | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/docs/release-checklist.rst b/docs/release-checklist.rst
index be32aea6c..18c908a99 100644
--- a/docs/release-checklist.rst
+++ b/docs/release-checklist.rst
@@ -97,10 +97,10 @@ they will need to evaluate which contributors' signatures they trust.
   - install each in a fresh virtualenv
   - run `tahoe` command
 - when satisfied, sign the tarballs:
-  - gpg --pinentry=loopback --armor --sign dist/tahoe_lafs-1.15.0rc0-py2-none-any.whl
-  - gpg --pinentry=loopback --armor --sign dist/tahoe_lafs-1.15.0rc0.tar.bz2
-  - gpg --pinentry=loopback --armor --sign dist/tahoe_lafs-1.15.0rc0.tar.gz
-  - gpg --pinentry=loopback --armor --sign dist/tahoe_lafs-1.15.0rc0.zip
+  - gpg --pinentry=loopback --armor --detach-sign dist/tahoe_lafs-1.15.0rc0-py2-none-any.whl
+  - gpg --pinentry=loopback --armor --detach-sign dist/tahoe_lafs-1.15.0rc0.tar.bz2
+  - gpg --pinentry=loopback --armor --detach-sign dist/tahoe_lafs-1.15.0rc0.tar.gz
+  - gpg --pinentry=loopback --armor --detach-sign dist/tahoe_lafs-1.15.0rc0.zip
 
 
 Privileged Contributor

From 116c59142d46a033cf4d9ac8615a43fee78cf022 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Tue, 12 Jan 2021 11:26:43 -0500
Subject: [PATCH 02/37] Port to Python 3.

---
 src/allmydata/test/web/test_logs.py | 4 ++++
 src/allmydata/util/_python3.py      | 1 +
 2 files changed, 5 insertions(+)

diff --git a/src/allmydata/test/web/test_logs.py b/src/allmydata/test/web/test_logs.py
index 4895ed6f0..ca8e5b918 100644
--- a/src/allmydata/test/web/test_logs.py
+++ b/src/allmydata/test/web/test_logs.py
@@ -9,6 +9,10 @@ from __future__ import (
     division,
 )
 
+from future.utils import PY2
+if PY2:
+    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
+
 from testtools.matchers import (
     Equals,
 )
diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py
index 14db70735..eca06058b 100644
--- a/src/allmydata/util/_python3.py
+++ b/src/allmydata/util/_python3.py
@@ -186,6 +186,7 @@ PORTED_TEST_MODULES = [
     "allmydata.test.test_util",
     "allmydata.test.web.test_common",
     "allmydata.test.web.test_grid",
+    "allmydata.test.web.test_logs",
     "allmydata.test.web.test_status",
     "allmydata.test.web.test_util",
     "allmydata.test.web.test_webish",

From d99c94753c1e222e0e5cb9eca79369f5f955f123 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Tue, 12 Jan 2021 11:38:37 -0500
Subject: [PATCH 03/37] On Python 3 we need to make sure bytes get written to
 the websocket.

---
 src/allmydata/web/logs.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/src/allmydata/web/logs.py b/src/allmydata/web/logs.py
index 0ba8b17e9..896dce418 100644
--- a/src/allmydata/web/logs.py
+++ b/src/allmydata/web/logs.py
@@ -5,6 +5,8 @@ from __future__ import (
     division,
 )
 
+from future.builtins import str
+
 import json
 
 from autobahn.twisted.resource import WebSocketResource
@@ -49,7 +51,11 @@ class TokenAuthenticatedWebSocketServerProtocol(WebSocketServerProtocol):
         """
         # probably want a try/except around here? what do we do if
         # transmission fails or anything else bad happens?
-        self.sendMessage(json.dumps(message))
+        encoded = json.dumps(message)
+        if isinstance(encoded, str):
+            # On Python 3 dumps() returns Unicode...
+            encoded = encoded.encode("utf-8")
+        self.sendMessage(encoded)
 
     def onOpen(self):
         """

From c2d69c53096fe1bd5894fd0cd198a475a2d592d5 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Tue, 12 Jan 2021 11:41:19 -0500
Subject: [PATCH 04/37] Merge all log tests into one test module.

---
 src/allmydata/test/test_websocket_logs.py | 54 ----------------------
 src/allmydata/test/web/test_logs.py       | 56 +++++++++++++++++++++++
 2 files changed, 56 insertions(+), 54 deletions(-)
 delete mode 100644 src/allmydata/test/test_websocket_logs.py

diff --git a/src/allmydata/test/test_websocket_logs.py b/src/allmydata/test/test_websocket_logs.py
deleted file mode 100644
index e666a4902..000000000
--- a/src/allmydata/test/test_websocket_logs.py
+++ /dev/null
@@ -1,54 +0,0 @@
-import json
-
-from twisted.trial import unittest
-from twisted.internet.defer import inlineCallbacks
-
-from eliot import log_call
-
-from autobahn.twisted.testing import create_memory_agent, MemoryReactorClockResolver, create_pumper
-
-from allmydata.web.logs import TokenAuthenticatedWebSocketServerProtocol
-
-
-class TestStreamingLogs(unittest.TestCase):
-    """
-    Test websocket streaming of logs
-    """
-
-    def setUp(self):
-        self.reactor = MemoryReactorClockResolver()
-        self.pumper = create_pumper()
-        self.agent = create_memory_agent(self.reactor, self.pumper, TokenAuthenticatedWebSocketServerProtocol)
-        return self.pumper.start()
-
-    def tearDown(self):
-        return self.pumper.stop()
-
-    @inlineCallbacks
-    def test_one_log(self):
-        """
-        write a single Eliot log and see it streamed via websocket
-        """
-
-        proto = yield self.agent.open(
-            transport_config=u"ws://localhost:1234/ws",
-            options={},
-        )
-
-        messages = []
-        def got_message(msg, is_binary=False):
-            messages.append(json.loads(msg))
-        proto.on("message", got_message)
-
-        @log_call(action_type=u"test:cli:some-exciting-action")
-        def do_a_thing():
-            pass
-
-        do_a_thing()
-
-        proto.transport.loseConnection()
-        yield proto.is_closed
-
-        self.assertEqual(len(messages), 2)
-        self.assertEqual("started", messages[0]["action_status"])
-        self.assertEqual("succeeded", messages[1]["action_status"])
diff --git a/src/allmydata/test/web/test_logs.py b/src/allmydata/test/web/test_logs.py
index ca8e5b918..5d697f910 100644
--- a/src/allmydata/test/web/test_logs.py
+++ b/src/allmydata/test/web/test_logs.py
@@ -1,5 +1,7 @@
 """
 Tests for ``allmydata.web.logs``.
+
+Ported to Python 3.
 """
 
 from __future__ import (
@@ -13,6 +15,15 @@ from future.utils import PY2
 if PY2:
     from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
+import json
+
+from twisted.trial import unittest
+from twisted.internet.defer import inlineCallbacks
+
+from eliot import log_call
+
+from autobahn.twisted.testing import create_memory_agent, MemoryReactorClockResolver, create_pumper
+
 from testtools.matchers import (
     Equals,
 )
@@ -41,6 +52,7 @@ from ..common import (
 
 from ...web.logs import (
     create_log_resources,
+    TokenAuthenticatedWebSocketServerProtocol,
 )
 
 class StreamingEliotLogsTests(SyncTestCase):
@@ -61,3 +73,47 @@ class StreamingEliotLogsTests(SyncTestCase):
             self.client.get(b"http:///v1"),
             succeeded(has_response_code(Equals(OK))),
         )
+
+
+class TestStreamingLogs(unittest.TestCase):
+    """
+    Test websocket streaming of logs
+    """
+
+    def setUp(self):
+        self.reactor = MemoryReactorClockResolver()
+        self.pumper = create_pumper()
+        self.agent = create_memory_agent(self.reactor, self.pumper, TokenAuthenticatedWebSocketServerProtocol)
+        return self.pumper.start()
+
+    def tearDown(self):
+        return self.pumper.stop()
+
+    @inlineCallbacks
+    def test_one_log(self):
+        """
+        write a single Eliot log and see it streamed via websocket
+        """
+
+        proto = yield self.agent.open(
+            transport_config=u"ws://localhost:1234/ws",
+            options={},
+        )
+
+        messages = []
+        def got_message(msg, is_binary=False):
+            messages.append(json.loads(msg))
+        proto.on("message", got_message)
+
+        @log_call(action_type=u"test:cli:some-exciting-action")
+        def do_a_thing():
+            pass
+
+        do_a_thing()
+
+        proto.transport.loseConnection()
+        yield proto.is_closed
+
+        self.assertEqual(len(messages), 2)
+        self.assertEqual("started", messages[0]["action_status"])
+        self.assertEqual("succeeded", messages[1]["action_status"])

From 7e5e3291381ddf03625375e718658566da009e65 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Tue, 12 Jan 2021 11:44:27 -0500
Subject: [PATCH 05/37] Port to Python 3.

---
 src/allmydata/util/_python3.py | 1 +
 src/allmydata/web/logs.py      | 5 +++--
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py
index eca06058b..9917be84c 100644
--- a/src/allmydata/util/_python3.py
+++ b/src/allmydata/util/_python3.py
@@ -113,6 +113,7 @@ PORTED_MODULES = [
     "allmydata.util.spans",
     "allmydata.util.statistics",
     "allmydata.util.time_format",
+    "allmydata.web.logs",
     "allmydata.webish",
 ]
 
diff --git a/src/allmydata/web/logs.py b/src/allmydata/web/logs.py
index 896dce418..6f15a3ca9 100644
--- a/src/allmydata/web/logs.py
+++ b/src/allmydata/web/logs.py
@@ -1,3 +1,6 @@
+"""
+Ported to Python 3.
+"""
 from __future__ import (
     print_function,
     unicode_literals,
@@ -5,8 +8,6 @@ from __future__ import (
     division,
 )
 
-from future.builtins import str
-
 import json
 
 from autobahn.twisted.resource import WebSocketResource

From bd364feec5c9d178929c37179fdc9066f65d97f1 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Tue, 12 Jan 2021 13:22:43 -0500
Subject: [PATCH 06/37] Tests pass on Python 3.

---
 src/allmydata/test/web/test_introducer.py | 2 +-
 src/allmydata/web/introweb.py             | 6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/allmydata/test/web/test_introducer.py b/src/allmydata/test/web/test_introducer.py
index 929fba507..43f4d5934 100644
--- a/src/allmydata/test/web/test_introducer.py
+++ b/src/allmydata/test/web/test_introducer.py
@@ -213,7 +213,7 @@ class IntroducerRootTests(unittest.TestCase):
         resource = IntroducerRoot(introducer_node)
         response = json.loads(
             self.successResultOf(
-                render(resource, {"t": [b"json"]}),
+                render(resource, {b"t": [b"json"]}),
             ),
         )
         self.assertEqual(
diff --git a/src/allmydata/web/introweb.py b/src/allmydata/web/introweb.py
index 6ec558e82..280d6cc26 100644
--- a/src/allmydata/web/introweb.py
+++ b/src/allmydata/web/introweb.py
@@ -26,10 +26,10 @@ class IntroducerRoot(MultiFormatResource):
         self.introducer_node = introducer_node
         self.introducer_service = introducer_node.getServiceNamed("introducer")
         # necessary as a root Resource
-        self.putChild("", self)
+        self.putChild(b"", self)
         static_dir = resource_filename("allmydata.web", "static")
         for filen in os.listdir(static_dir):
-            self.putChild(filen, static.File(os.path.join(static_dir, filen)))
+            self.putChild(filen.encode("utf-8"), static.File(os.path.join(static_dir, filen)))
 
     def _create_element(self):
         """
@@ -66,7 +66,7 @@ class IntroducerRoot(MultiFormatResource):
             announcement_summary[service_name] += 1
         res[u"announcement_summary"] = announcement_summary
 
-        return json.dumps(res, indent=1) + b"\n"
+        return (json.dumps(res, indent=1) + "\n").encode("utf-8")
 
 
 class IntroducerRootElement(Element):

From 8c41f60fdb2b6017f83497ed7a877f955fd7f673 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Tue, 12 Jan 2021 14:17:38 -0500
Subject: [PATCH 07/37] Port to Python 3.

---
 src/allmydata/test/web/test_introducer.py | 12 ++++++++++++
 src/allmydata/util/_python3.py            |  1 +
 2 files changed, 13 insertions(+)

diff --git a/src/allmydata/test/web/test_introducer.py b/src/allmydata/test/web/test_introducer.py
index 43f4d5934..08d95bda9 100644
--- a/src/allmydata/test/web/test_introducer.py
+++ b/src/allmydata/test/web/test_introducer.py
@@ -1,3 +1,15 @@
+"""
+Ported to Python 3.
+"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from future.utils import PY2
+if PY2:
+    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
+
 import json
 from os.path import join
 
diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py
index 9917be84c..63aa5bb0a 100644
--- a/src/allmydata/util/_python3.py
+++ b/src/allmydata/util/_python3.py
@@ -187,6 +187,7 @@ PORTED_TEST_MODULES = [
     "allmydata.test.test_util",
     "allmydata.test.web.test_common",
     "allmydata.test.web.test_grid",
+    "allmydata.test.web.test_introducer",
     "allmydata.test.web.test_logs",
     "allmydata.test.web.test_status",
     "allmydata.test.web.test_util",

From c076e1ee2646758a65c42100265e4ddb80e269ef Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Tue, 12 Jan 2021 14:24:11 -0500
Subject: [PATCH 08/37] Just fix all the putChild.

---
 src/allmydata/web/root.py | 21 +++++++++++----------
 1 file changed, 11 insertions(+), 10 deletions(-)

diff --git a/src/allmydata/web/root.py b/src/allmydata/web/root.py
index fdc72ab71..f6316bff5 100644
--- a/src/allmydata/web/root.py
+++ b/src/allmydata/web/root.py
@@ -1,3 +1,5 @@
+from past.builtins import unicode
+
 import os
 import time
 import urllib
@@ -227,26 +229,25 @@ class Root(MultiFormatResource):
         self._client = client
         self._now_fn = now_fn
 
-        # Children need to be bytes; for now just doing these to make specific
-        # tests pass on Python 3, but eventually will do all them when this
-        # module is ported to Python 3 (if not earlier).
         self.putChild(b"uri", URIHandler(client))
-        self.putChild("cap", URIHandler(client))
+        self.putChild(b"cap", URIHandler(client))
 
         # Handler for everything beneath "/private", an area of the resource
         # hierarchy which is only accessible with the private per-node API
         # auth token.
-        self.putChild("private", create_private_tree(client.get_auth_token))
+        self.putChild(b"private", create_private_tree(client.get_auth_token))
 
-        self.putChild("file", FileHandler(client))
-        self.putChild("named", FileHandler(client))
-        self.putChild("status", status.Status(client.get_history()))
-        self.putChild("statistics", status.Statistics(client.stats_provider))
+        self.putChild(b"file", FileHandler(client))
+        self.putChild(b"named", FileHandler(client))
+        self.putChild(b"status", status.Status(client.get_history()))
+        self.putChild(b"statistics", status.Statistics(client.stats_provider))
         static_dir = resource_filename("allmydata.web", "static")
         for filen in os.listdir(static_dir):
+            if isinstance(filen, unicode):
+                filen = filen.encode("utf-8")
             self.putChild(filen, static.File(os.path.join(static_dir, filen)))
 
-        self.putChild("report_incident", IncidentReporter())
+        self.putChild(b"report_incident", IncidentReporter())
 
     @exception_to_child
     def getChild(self, path, request):

From 4940da47da1cd98ae9e0de3b5b80d11a8c2f573a Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Tue, 12 Jan 2021 14:24:17 -0500
Subject: [PATCH 09/37] Tests pass on Python 3.

---
 src/allmydata/test/web/test_private.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/src/allmydata/test/web/test_private.py b/src/allmydata/test/web/test_private.py
index 27ddbcf78..583f629f5 100644
--- a/src/allmydata/test/web/test_private.py
+++ b/src/allmydata/test/web/test_private.py
@@ -9,6 +9,8 @@ from __future__ import (
     division,
 )
 
+from future.builtins import str
+
 from testtools.matchers import (
     Equals,
 )
@@ -56,6 +58,7 @@ class PrivacyTests(SyncTestCase):
         return super(PrivacyTests, self).setUp()
 
     def _authorization(self, scheme, value):
+        value = str(value, "utf-8")
         return Headers({
             u"authorization": [u"{} {}".format(scheme, value)],
         })
@@ -90,7 +93,7 @@ class PrivacyTests(SyncTestCase):
         self.assertThat(
             self.client.head(
                 b"http:///foo/bar",
-                headers=self._authorization(SCHEME, u"foo bar"),
+                headers=self._authorization(str(SCHEME, "utf-8"), b"foo bar"),
             ),
             succeeded(has_response_code(Equals(UNAUTHORIZED))),
         )
@@ -103,7 +106,7 @@ class PrivacyTests(SyncTestCase):
         self.assertThat(
             self.client.head(
                 b"http:///foo/bar",
-                headers=self._authorization(SCHEME, self.token),
+                headers=self._authorization(str(SCHEME, "utf-8"), self.token),
             ),
             # It's a made up URL so we don't get a 200, either, but a 404.
             succeeded(has_response_code(Equals(NOT_FOUND))),

From 03fb936716689447edb676c85044cabd79833f3d Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Tue, 12 Jan 2021 14:25:16 -0500
Subject: [PATCH 10/37] Port to Python 3.

---
 src/allmydata/test/web/test_private.py | 6 ++++++
 src/allmydata/util/_python3.py         | 1 +
 2 files changed, 7 insertions(+)

diff --git a/src/allmydata/test/web/test_private.py b/src/allmydata/test/web/test_private.py
index 583f629f5..293796b1c 100644
--- a/src/allmydata/test/web/test_private.py
+++ b/src/allmydata/test/web/test_private.py
@@ -1,5 +1,7 @@
 """
 Tests for ``allmydata.web.private``.
+
+Ported to Python 3.
 """
 
 from __future__ import (
@@ -9,6 +11,10 @@ from __future__ import (
     division,
 )
 
+from future.utils import PY2
+if PY2:
+    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
+
 from future.builtins import str
 
 from testtools.matchers import (
diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py
index 63aa5bb0a..7cd80335c 100644
--- a/src/allmydata/util/_python3.py
+++ b/src/allmydata/util/_python3.py
@@ -189,6 +189,7 @@ PORTED_TEST_MODULES = [
     "allmydata.test.web.test_grid",
     "allmydata.test.web.test_introducer",
     "allmydata.test.web.test_logs",
+    "allmydata.test.web.test_private",
     "allmydata.test.web.test_status",
     "allmydata.test.web.test_util",
     "allmydata.test.web.test_webish",

From 7a3e9ab43e520a531ab63178a06a1fe0160d6aed Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Tue, 12 Jan 2021 14:39:20 -0500
Subject: [PATCH 11/37] Tests pass on Python 3.

---
 src/allmydata/test/web/test_root.py | 8 ++++----
 src/allmydata/web/root.py           | 4 ++--
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/allmydata/test/web/test_root.py b/src/allmydata/test/web/test_root.py
index 0715c8102..497440d1c 100644
--- a/src/allmydata/test/web/test_root.py
+++ b/src/allmydata/test/web/test_root.py
@@ -1,6 +1,6 @@
 import time
 
-from urllib import (
+from urllib.parse import (
     quote,
 )
 
@@ -77,7 +77,7 @@ class RenderSlashUri(unittest.TestCase):
         )
         self.assertEqual(
             response_body,
-            "Invalid capability",
+            b"Invalid capability",
         )
 
 
@@ -92,7 +92,7 @@ class RenderServiceRow(unittest.TestCase):
         ann = {"anonymous-storage-FURL": "pb://w2hqnbaa25yw4qgcvghl5psa3srpfgw3@tcp:127.0.0.1:51309/vucto2z4fxment3vfxbqecblbf6zyp6x",
                "permutation-seed-base32": "w2hqnbaa25yw4qgcvghl5psa3srpfgw3",
                }
-        srv = NativeStorageServer("server_id", ann, None, {}, EMPTY_CLIENT_CONFIG)
+        srv = NativeStorageServer(b"server_id", ann, None, {}, EMPTY_CLIENT_CONFIG)
         srv.get_connection_status = lambda: ConnectionStatus(False, "summary", {}, 0, 0)
 
         class FakeClient(_Client):
@@ -103,7 +103,7 @@ class RenderServiceRow(unittest.TestCase):
                     tub_maker=None,
                     node_config=EMPTY_CLIENT_CONFIG,
                 )
-                self.storage_broker.test_add_server("test-srv", srv)
+                self.storage_broker.test_add_server(b"test-srv", srv)
 
         root = RootElement(FakeClient(), time.time)
         req = DummyRequest(b"")
diff --git a/src/allmydata/web/root.py b/src/allmydata/web/root.py
index f6316bff5..5829da51e 100644
--- a/src/allmydata/web/root.py
+++ b/src/allmydata/web/root.py
@@ -2,7 +2,7 @@ from past.builtins import unicode
 
 import os
 import time
-import urllib
+from urllib.parse import quote as urlquote
 
 from hyperlink import DecodedURL, URL
 from pkg_resources import resource_filename
@@ -83,7 +83,7 @@ class URIHandler(resource.Resource, object):
         # it seems Nevow was creating absolute URLs including
         # host/port whereas req.uri is absolute (but lacks host/port)
         redir_uri = URL.from_text(req.prePathURL().decode('utf8'))
-        redir_uri = redir_uri.child(urllib.quote(uri_arg).decode('utf8'))
+        redir_uri = redir_uri.child(urlquote(uri_arg))
         # add back all the query args that AREN'T "?uri="
         for k, values in req.args.items():
             if k != b"uri":

From 5d77282784b88dbcb3445d279b9c9d0afd6f6db2 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Tue, 12 Jan 2021 14:40:33 -0500
Subject: [PATCH 12/37] Ported to Python 3.

---
 src/allmydata/test/web/test_root.py | 12 ++++++++++++
 src/allmydata/util/_python3.py      |  1 +
 2 files changed, 13 insertions(+)

diff --git a/src/allmydata/test/web/test_root.py b/src/allmydata/test/web/test_root.py
index 497440d1c..ca3cc695d 100644
--- a/src/allmydata/test/web/test_root.py
+++ b/src/allmydata/test/web/test_root.py
@@ -1,3 +1,15 @@
+"""
+Ported to Python 3.
+"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from future.utils import PY2
+if PY2:
+    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
+
 import time
 
 from urllib.parse import (
diff --git a/src/allmydata/util/_python3.py b/src/allmydata/util/_python3.py
index 7cd80335c..575a69cb7 100644
--- a/src/allmydata/util/_python3.py
+++ b/src/allmydata/util/_python3.py
@@ -190,6 +190,7 @@ PORTED_TEST_MODULES = [
     "allmydata.test.web.test_introducer",
     "allmydata.test.web.test_logs",
     "allmydata.test.web.test_private",
+    "allmydata.test.web.test_root",
     "allmydata.test.web.test_status",
     "allmydata.test.web.test_util",
     "allmydata.test.web.test_webish",

From 6b0849490ad8cbbb23041b16e3850be6e837d30b Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Tue, 12 Jan 2021 14:40:46 -0500
Subject: [PATCH 13/37] News file.

---
 newsfragments/3589.minor | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 create mode 100644 newsfragments/3589.minor

diff --git a/newsfragments/3589.minor b/newsfragments/3589.minor
new file mode 100644
index 000000000..e69de29bb

From aace119790f92f691c586067f3554a842656c148 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Wed, 13 Jan 2021 09:55:54 -0500
Subject: [PATCH 14/37] Fix Python 3 issue with combining bytes and unicode.

---
 src/allmydata/web/root.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/allmydata/web/root.py b/src/allmydata/web/root.py
index 5829da51e..e3d49cd66 100644
--- a/src/allmydata/web/root.py
+++ b/src/allmydata/web/root.py
@@ -243,9 +243,10 @@ class Root(MultiFormatResource):
         self.putChild(b"statistics", status.Statistics(client.stats_provider))
         static_dir = resource_filename("allmydata.web", "static")
         for filen in os.listdir(static_dir):
+            child_path = filen
             if isinstance(filen, unicode):
-                filen = filen.encode("utf-8")
-            self.putChild(filen, static.File(os.path.join(static_dir, filen)))
+                child_path = filen.encode("utf-8")
+            self.putChild(child_path, static.File(os.path.join(static_dir, filen)))
 
         self.putChild(b"report_incident", IncidentReporter())
 

From c5669e16e0ba0943e83fd7f3e39a2569eb582743 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Wed, 13 Jan 2021 09:56:08 -0500
Subject: [PATCH 15/37] Fix flake.

---
 src/allmydata/test/web/test_private.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/src/allmydata/test/web/test_private.py b/src/allmydata/test/web/test_private.py
index 293796b1c..b426b4d93 100644
--- a/src/allmydata/test/web/test_private.py
+++ b/src/allmydata/test/web/test_private.py
@@ -15,8 +15,6 @@ from future.utils import PY2
 if PY2:
     from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
 
-from future.builtins import str
-
 from testtools.matchers import (
     Equals,
 )

From a2dab7c89fed217603116b97fd7d5ca8d0426823 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Thu, 14 Jan 2021 09:40:10 -0500
Subject: [PATCH 16/37] Only do this on Python 3.

---
 src/allmydata/web/root.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/allmydata/web/root.py b/src/allmydata/web/root.py
index e3d49cd66..b1ba501a9 100644
--- a/src/allmydata/web/root.py
+++ b/src/allmydata/web/root.py
@@ -1,3 +1,4 @@
+from future.utils import PY3
 from past.builtins import unicode
 
 import os
@@ -244,7 +245,7 @@ class Root(MultiFormatResource):
         static_dir = resource_filename("allmydata.web", "static")
         for filen in os.listdir(static_dir):
             child_path = filen
-            if isinstance(filen, unicode):
+            if PY3:
                 child_path = filen.encode("utf-8")
             self.putChild(child_path, static.File(os.path.join(static_dir, filen)))
 

From 42b31a28099ed7f5c732c4bfc6a71db46b303559 Mon Sep 17 00:00:00 2001
From: Itamar Turner-Trauring <itamar@itamarst.org>
Date: Thu, 14 Jan 2021 15:58:18 -0500
Subject: [PATCH 17/37] Fix flake.

---
 src/allmydata/web/root.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/allmydata/web/root.py b/src/allmydata/web/root.py
index b1ba501a9..0ef6b00d2 100644
--- a/src/allmydata/web/root.py
+++ b/src/allmydata/web/root.py
@@ -1,5 +1,4 @@
 from future.utils import PY3
-from past.builtins import unicode
 
 import os
 import time

From 621de4d882a8a0df0bb555897380dc9e7f111302 Mon Sep 17 00:00:00 2001
From: Sajith Sasidharan <sajith@hcoop.net>
Date: Mon, 18 Jan 2021 10:55:18 -0500
Subject: [PATCH 18/37] Add newsfragment

---
 newsfragments/3591.minor | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 create mode 100644 newsfragments/3591.minor

diff --git a/newsfragments/3591.minor b/newsfragments/3591.minor
new file mode 100644
index 000000000..e69de29bb

From fa1a8e8371a2a6567901e8b5dcd7259f0e1e7352 Mon Sep 17 00:00:00 2001
From: Sajith Sasidharan <sajith@hcoop.net>
Date: Mon, 18 Jan 2021 10:57:31 -0500
Subject: [PATCH 19/37] Upgrade pip used in GitHub Actions

From pip 20.1+ onward, "pip cache dir" can be used to find location of
pip cache, and this is useful across all three major OSes supported by
GitHub Actions.
---
 .github/workflows/ci.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index fd5049104..50a1a3d2f 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -43,7 +43,7 @@ jobs:
 
       - name: Install Python packages
         run: |
-          pip install --upgrade codecov tox setuptools
+          pip install --upgrade codecov tox setuptools pip
           pip list
 
       - name: Display tool versions
@@ -114,7 +114,7 @@ jobs:
 
       - name: Install Python packages
         run: |
-          pip install --upgrade tox
+          pip install --upgrade tox pip
           pip list
 
       - name: Display tool versions
@@ -166,7 +166,7 @@ jobs:
 
       - name: Install Python packages
         run: |
-          pip install --upgrade tox
+          pip install --upgrade tox pip
           pip list
 
       - name: Display tool versions

From 27a122088cdba445465428da6e7680c9a22e6c74 Mon Sep 17 00:00:00 2001
From: Sajith Sasidharan <sajith@hcoop.net>
Date: Mon, 18 Jan 2021 11:02:55 -0500
Subject: [PATCH 20/37] Use pip cache on GitHub Actions

Using the method outlined in https://github.com/actions/cache/blob/main/examples.md#using-pip-to-get-cache-location
---
 .github/workflows/ci.yml | 39 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 39 insertions(+)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 50a1a3d2f..8ca015e07 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -46,6 +46,19 @@ jobs:
           pip install --upgrade codecov tox setuptools pip
           pip list
 
+      - name: Get pip cache directory
+        id: pip-cache
+        run: |
+          echo "::set-output name=dir::$(pip cache dir)"
+
+      - name: pip cache
+        uses: actions/cache@v2
+        with:
+          path: ${{ steps.pip-cache.outputs.dir }}
+          key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+
       - name: Display tool versions
         run: python misc/build_helpers/show-tool-versions.py
 
@@ -117,6 +130,19 @@ jobs:
           pip install --upgrade tox pip
           pip list
 
+      - name: Get pip cache directory
+        id: pip-cache
+        run: |
+          echo "::set-output name=dir::$(pip cache dir)"
+
+      - name: pip cache
+        uses: actions/cache@v2
+        with:
+          path: ${{ steps.pip-cache.outputs.dir }}
+          key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+
       - name: Display tool versions
         run: python misc/build_helpers/show-tool-versions.py
 
@@ -169,6 +195,19 @@ jobs:
           pip install --upgrade tox pip
           pip list
 
+      - name: Get pip cache directory
+        id: pip-cache
+        run: |
+          echo "::set-output name=dir::$(pip cache dir)"
+
+      - name: pip cache
+        uses: actions/cache@v2
+        with:
+          path: ${{ steps.pip-cache.outputs.dir }}
+          key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+
       - name: Display tool versions
         run: python misc/build_helpers/show-tool-versions.py
 

From 573ab8768b4443a8e41304708040607974a9f4d4 Mon Sep 17 00:00:00 2001
From: Sajith Sasidharan <sajith@hcoop.net>
Date: Mon, 18 Jan 2021 11:08:56 -0500
Subject: [PATCH 21/37] Re-title "use pip cache" step in GitHub Actions

---
 .github/workflows/ci.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 8ca015e07..51cec9e54 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -51,7 +51,7 @@ jobs:
         run: |
           echo "::set-output name=dir::$(pip cache dir)"
 
-      - name: pip cache
+      - name: Use pip cache
         uses: actions/cache@v2
         with:
           path: ${{ steps.pip-cache.outputs.dir }}
@@ -135,7 +135,7 @@ jobs:
         run: |
           echo "::set-output name=dir::$(pip cache dir)"
 
-      - name: pip cache
+      - name: Use pip cache
         uses: actions/cache@v2
         with:
           path: ${{ steps.pip-cache.outputs.dir }}
@@ -200,7 +200,7 @@ jobs:
         run: |
           echo "::set-output name=dir::$(pip cache dir)"
 
-      - name: pip cache
+      - name: Use pip cache
         uses: actions/cache@v2
         with:
           path: ${{ steps.pip-cache.outputs.dir }}

From f731159cd7608925c5713cb4053067ef81c6bf8b Mon Sep 17 00:00:00 2001
From: Sajith Sasidharan <sajith@hcoop.net>
Date: Mon, 18 Jan 2021 13:13:32 -0500
Subject: [PATCH 22/37] Install Python packages after setting up pip cache

---
 .github/workflows/ci.yml | 30 +++++++++++++++---------------
 1 file changed, 15 insertions(+), 15 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 51cec9e54..c3f32c919 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -41,11 +41,6 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
 
-      - name: Install Python packages
-        run: |
-          pip install --upgrade codecov tox setuptools pip
-          pip list
-
       - name: Get pip cache directory
         id: pip-cache
         run: |
@@ -59,6 +54,11 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-pip-
 
+      - name: Install Python packages
+        run: |
+          pip install --upgrade codecov tox setuptools pip
+          pip list
+
       - name: Display tool versions
         run: python misc/build_helpers/show-tool-versions.py
 
@@ -125,11 +125,6 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
 
-      - name: Install Python packages
-        run: |
-          pip install --upgrade tox pip
-          pip list
-
       - name: Get pip cache directory
         id: pip-cache
         run: |
@@ -143,6 +138,11 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-pip-
 
+      - name: Install Python packages
+        run: |
+          pip install --upgrade tox pip
+          pip list
+
       - name: Display tool versions
         run: python misc/build_helpers/show-tool-versions.py
 
@@ -190,11 +190,6 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
 
-      - name: Install Python packages
-        run: |
-          pip install --upgrade tox pip
-          pip list
-
       - name: Get pip cache directory
         id: pip-cache
         run: |
@@ -208,6 +203,11 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-pip-
 
+      - name: Install Python packages
+        run: |
+          pip install --upgrade tox pip
+          pip list
+
       - name: Display tool versions
         run: python misc/build_helpers/show-tool-versions.py
 

From 8bf068f99152aed31b51ab86b6f3edbfd20baf4a Mon Sep 17 00:00:00 2001
From: Sajith Sasidharan <sajith@hcoop.net>
Date: Mon, 18 Jan 2021 13:19:18 -0500
Subject: [PATCH 23/37] What's the pip version on GitHub Actions?

There's no need of upgrading pip if GA offers a sufficiently new pip.
---
 .github/workflows/ci.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index c3f32c919..0135d9ab9 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -44,6 +44,7 @@ jobs:
       - name: Get pip cache directory
         id: pip-cache
         run: |
+          pip --version
           echo "::set-output name=dir::$(pip cache dir)"
 
       - name: Use pip cache

From 1f1a30095ea9359a3c3b752bfbc513efa3685dca Mon Sep 17 00:00:00 2001
From: Sajith Sasidharan <sajith@hcoop.net>
Date: Mon, 18 Jan 2021 13:22:04 -0500
Subject: [PATCH 24/37] Get pip version for all three GitHub Actions OSes

---
 .github/workflows/ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 0135d9ab9..1943933b2 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -44,7 +44,6 @@ jobs:
       - name: Get pip cache directory
         id: pip-cache
         run: |
-          pip --version
           echo "::set-output name=dir::$(pip cache dir)"
 
       - name: Use pip cache
@@ -194,6 +193,7 @@ jobs:
       - name: Get pip cache directory
         id: pip-cache
         run: |
+          pip --version
           echo "::set-output name=dir::$(pip cache dir)"
 
       - name: Use pip cache

From 2a1a5cb0a5b232487894dafdfa7200a96f1b1d4b Mon Sep 17 00:00:00 2001
From: Sajith Sasidharan <sajith@hcoop.net>
Date: Mon, 18 Jan 2021 13:32:53 -0500
Subject: [PATCH 25/37] GitHub Actions have sufficiently recent pip

At the time of writing this commit message, GitHub Actions offers pip
v20.3.3 for both ubuntu-latest and windows-latest, and pip v20.3.1 for
macos-latest.  Those are sufficiently recent pip versions that have
"cache dir" sub-command.
---
 .github/workflows/ci.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 1943933b2..c3f32c919 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -193,7 +193,6 @@ jobs:
       - name: Get pip cache directory
         id: pip-cache
         run: |
-          pip --version
           echo "::set-output name=dir::$(pip cache dir)"
 
       - name: Use pip cache

From adbe23fe7aac3a0f8b55b3135d266a531ec0bcc4 Mon Sep 17 00:00:00 2001
From: Sajith Sasidharan <sajith@hcoop.net>
Date: Mon, 18 Jan 2021 13:34:25 -0500
Subject: [PATCH 26/37] Add a note about pip version on GitHub Actions

---
 .github/workflows/ci.yml | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index c3f32c919..8ccf07aba 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -41,6 +41,11 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
 
+      # We need "pip cache dir", which became a thing in pip v20.1+.
+      # At the time of writing this, GitHub Actions offers pip v20.3.3
+      # for both ubuntu-latest and windows-latest, and pip v20.3.1 for
+      # macos-latest.  Those are sufficiently recent pip versions that
+      # have "cache dir" sub-command.
       - name: Get pip cache directory
         id: pip-cache
         run: |

From 99cca0ea8e7223886b5ee9d184dd70a70141a031 Mon Sep 17 00:00:00 2001
From: Sajith Sasidharan <sajith@hcoop.net>
Date: Mon, 18 Jan 2021 13:38:09 -0500
Subject: [PATCH 27/37] No need of upgrading pip on GitHub Actions

---
 .github/workflows/ci.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 8ccf07aba..afb99c67a 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -61,7 +61,7 @@ jobs:
 
       - name: Install Python packages
         run: |
-          pip install --upgrade codecov tox setuptools pip
+          pip install --upgrade codecov tox setuptools
           pip list
 
       - name: Display tool versions
@@ -145,7 +145,7 @@ jobs:
 
       - name: Install Python packages
         run: |
-          pip install --upgrade tox pip
+          pip install --upgrade tox
           pip list
 
       - name: Display tool versions
@@ -210,7 +210,7 @@ jobs:
 
       - name: Install Python packages
         run: |
-          pip install --upgrade tox pip
+          pip install --upgrade tox
           pip list
 
       - name: Display tool versions

From 9e4ea0c4910c819284b9fc74fd488d522bcc5d47 Mon Sep 17 00:00:00 2001
From: Sajith Sasidharan <sajith@hcoop.net>
Date: Mon, 18 Jan 2021 17:54:17 -0500
Subject: [PATCH 28/37] Use fetch-depth of 0 with GitHub Actions

Using a fetch-depth of 0 should have the same effect as as `git fetch
--prune --unshallow` after doing a shallow checkout.
---
 .github/workflows/ci.yml | 17 ++++++++---------
 1 file changed, 8 insertions(+), 9 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index afb99c67a..213a87ba1 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -30,11 +30,12 @@ jobs:
         with:
           args: install vcpython27
 
+      # See https://github.com/actions/checkout. A fetch-depth of 0
+      # fetches all tags and branches.
       - name: Check out Tahoe-LAFS sources
         uses: actions/checkout@v2
-
-      - name: Fetch all history for all tags and branches
-        run: git fetch --prune --unshallow
+        with:
+          fetch-depth: 0
 
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v1
@@ -121,9 +122,8 @@ jobs:
 
       - name: Check out Tahoe-LAFS sources
         uses: actions/checkout@v2
-
-      - name: Fetch all history for all tags and branches
-        run: git fetch --prune --unshallow
+        with:
+          fetch-depth: 0
 
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v1
@@ -186,9 +186,8 @@ jobs:
 
       - name: Check out Tahoe-LAFS sources
         uses: actions/checkout@v2
-
-      - name: Fetch all history for all tags and branches
-        run: git fetch --prune --unshallow
+        with:
+          fetch-depth: 0
 
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v1

From ed92202762f56c7b37a161064dc4613ec7d9d8d7 Mon Sep 17 00:00:00 2001
From: Sajith Sasidharan <sajith@hcoop.net>
Date: Mon, 18 Jan 2021 17:55:07 -0500
Subject: [PATCH 29/37] Updates comments about GitHub cache action

---
 .github/workflows/ci.yml | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 213a87ba1..ee36833ca 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -42,16 +42,17 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
 
-      # We need "pip cache dir", which became a thing in pip v20.1+.
-      # At the time of writing this, GitHub Actions offers pip v20.3.3
-      # for both ubuntu-latest and windows-latest, and pip v20.3.1 for
-      # macos-latest.  Those are sufficiently recent pip versions that
-      # have "cache dir" sub-command.
+      # To use pip caching with GitHub Actions in an OS-independent
+      # manner, we need `pip cache dir` command, which became
+      # available since pip v20.1+.  At the time of writing this,
+      # GitHub Actions offers pip v20.3.3 for both ubuntu-latest and
+      # windows-latest, and pip v20.3.1 for macos-latest.
       - name: Get pip cache directory
         id: pip-cache
         run: |
           echo "::set-output name=dir::$(pip cache dir)"
 
+      # See https://github.com/actions/cache
       - name: Use pip cache
         uses: actions/cache@v2
         with:

From 5dd7aa2dfda61156f83e24c37312238a631757c1 Mon Sep 17 00:00:00 2001
From: meejah <meejah@meejah.ca>
Date: Tue, 19 Jan 2021 11:25:09 -0700
Subject: [PATCH 30/37] news

---
 newsfragments/2920.minor | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 create mode 100644 newsfragments/2920.minor

diff --git a/newsfragments/2920.minor b/newsfragments/2920.minor
new file mode 100644
index 000000000..e69de29bb

From 8be3678cb47f0902a94d2ed1b1d651842b738efd Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Wed, 20 Jan 2021 11:22:22 -0500
Subject: [PATCH 31/37] Directly test read_encrypted behavior

---
 src/allmydata/test/test_upload.py | 69 +++++++++++++++++++++++++++++++
 1 file changed, 69 insertions(+)

diff --git a/src/allmydata/test/test_upload.py b/src/allmydata/test/test_upload.py
index 94d7575c3..7e41bfc24 100644
--- a/src/allmydata/test/test_upload.py
+++ b/src/allmydata/test/test_upload.py
@@ -14,6 +14,17 @@ if PY2:
 
 import os, shutil
 from io import BytesIO
+from base64 import (
+    b64encode,
+)
+
+from hypothesis import (
+    given,
+)
+from hypothesis.strategies import (
+    just,
+    integers,
+)
 
 from twisted.trial import unittest
 from twisted.python.failure import Failure
@@ -2029,6 +2040,64 @@ class EncodingParameters(GridTestMixin, unittest.TestCase, SetDEPMixin,
         f.close()
         return None
 
+
+class EncryptAnUploadableTests(unittest.TestCase):
+    """
+    Tests for ``EncryptAnUploadable``.
+    """
+    def test_same_length(self):
+        """
+        ``EncryptAnUploadable.read_encrypted`` returns ciphertext of the same
+        length as the underlying plaintext.
+        """
+        plaintext = b"hello world"
+        uploadable = upload.FileHandle(BytesIO(plaintext), None)
+        uploadable.set_default_encoding_parameters({
+            # These values shouldn't matter.
+            "k": 3,
+            "happy": 5,
+            "n": 10,
+            "max_segment_size": 128 * 1024,
+        })
+        encrypter = upload.EncryptAnUploadable(uploadable)
+        ciphertext = b"".join(self.successResultOf(encrypter.read_encrypted(1024, False)))
+        self.assertEqual(len(ciphertext), len(plaintext))
+
+    @given(just(b"hello world"), integers(min_value=0, max_value=len(b"hello world")))
+    def test_known_result(self, plaintext, split_at):
+        """
+        ``EncryptAnUploadable.read_encrypted`` returns a known-correct ciphertext
+        string for certain inputs.  The ciphertext is independent of the read
+        sizes.
+        """
+        convergence = b"\x42" * 16
+        uploadable = upload.FileHandle(BytesIO(plaintext), convergence)
+        uploadable.set_default_encoding_parameters({
+            # The convergence key is a function of k, n, and max_segment_size
+            # (among other things).  The value for happy doesn't matter
+            # though.
+            "k": 3,
+            "happy": 5,
+            "n": 10,
+            "max_segment_size": 128 * 1024,
+        })
+        encrypter = upload.EncryptAnUploadable(uploadable)
+        def read(n):
+            return b"".join(self.successResultOf(encrypter.read_encrypted(n, False)))
+
+        # Read the string in one or two pieces to make sure underlying state
+        # is maintained properly.
+        first = read(split_at)
+        second = read(len(plaintext) - split_at)
+        third = read(1)
+        ciphertext = first + second + third
+
+        self.assertEqual(
+            b"Jd2LHCRXozwrEJc=",
+            b64encode(ciphertext),
+        )
+
+
 # TODO:
 #  upload with exactly 75 servers (shares_of_happiness)
 #  have a download fail

From f75f71cba6e98ac508cf06108523a9d0c1a4842f Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Wed, 20 Jan 2021 11:23:35 -0500
Subject: [PATCH 32/37] news fragment

---
 newsfragments/3594.minor | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 create mode 100644 newsfragments/3594.minor

diff --git a/newsfragments/3594.minor b/newsfragments/3594.minor
new file mode 100644
index 000000000..e69de29bb

From 932481ad47c650cb00070394829a7cc268fdd00e Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Wed, 20 Jan 2021 12:58:03 -0500
Subject: [PATCH 33/37] A helper for doing something repeatedly for a while

---
 src/allmydata/test/test_deferredutil.py | 55 +++++++++++++++++++++++++
 src/allmydata/util/deferredutil.py      | 30 ++++++++++++++
 2 files changed, 85 insertions(+)

diff --git a/src/allmydata/test/test_deferredutil.py b/src/allmydata/test/test_deferredutil.py
index 6ebc93556..2a155089f 100644
--- a/src/allmydata/test/test_deferredutil.py
+++ b/src/allmydata/test/test_deferredutil.py
@@ -74,3 +74,58 @@ class DeferredUtilTests(unittest.TestCase, deferredutil.WaitForDelayedCallsMixin
         d = defer.succeed(None)
         d.addBoth(self.wait_for_delayed_calls)
         return d
+
+
+class UntilTests(unittest.TestCase):
+    """
+    Tests for ``deferredutil.until``.
+    """
+    def test_exception(self):
+        """
+        If the action raises an exception, the ``Deferred`` returned by ``until``
+        fires with a ``Failure``.
+        """
+        self.assertFailure(
+            deferredutil.until(lambda: 1/0, lambda: True),
+            ZeroDivisionError,
+        )
+
+    def test_stops_on_condition(self):
+        """
+        The action is called repeatedly until ``condition`` returns ``True``.
+        """
+        calls = []
+        def action():
+            calls.append(None)
+
+        def condition():
+            return len(calls) == 3
+
+        self.assertIs(
+            self.successResultOf(
+                deferredutil.until(action, condition),
+            ),
+            None,
+        )
+        self.assertEqual(3, len(calls))
+
+    def test_waits_for_deferred(self):
+        """
+        If the action returns a ``Deferred`` then it is called again when the
+        ``Deferred`` fires.
+        """
+        counter = [0]
+        r1 = defer.Deferred()
+        r2 = defer.Deferred()
+        results = [r1, r2]
+        def action():
+            counter[0] += 1
+            return results.pop(0)
+
+        def condition():
+            return False
+
+        deferredutil.until(action, condition)
+        self.assertEqual([1], counter)
+        r1.callback(None)
+        self.assertEqual([2], counter)
diff --git a/src/allmydata/util/deferredutil.py b/src/allmydata/util/deferredutil.py
index 1d13f61e6..ed2a11ee4 100644
--- a/src/allmydata/util/deferredutil.py
+++ b/src/allmydata/util/deferredutil.py
@@ -15,7 +15,18 @@ if PY2:
 
 import time
 
+try:
+    from typing import (
+        Callable,
+        Any,
+    )
+except ImportError:
+    pass
+
 from foolscap.api import eventually
+from eliot.twisted import (
+    inline_callbacks,
+)
 from twisted.internet import defer, reactor, error
 from twisted.python.failure import Failure
 
@@ -201,3 +212,22 @@ class WaitForDelayedCallsMixin(PollMixin):
         d.addErrback(log.err, "error while waiting for delayed calls")
         d.addBoth(lambda ign: res)
         return d
+
+@inline_callbacks
+def until(
+        action,     # type: Callable[[], defer.Deferred[Any]]
+        condition,  # type: Callable[[], bool]
+):
+    # type: (...) -> defer.Deferred[None]
+    """
+    Run a Deferred-returning function until a condition is true.
+
+    :param action: The action to run.
+    :param condition: The predicate signaling stop.
+
+    :return: A Deferred that fires after the condition signals stop.
+    """
+    while True:
+        yield action()
+        if condition():
+            break

From 12087738d682c051aeeb75a7c90dd78d7b8ebfb0 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Wed, 20 Jan 2021 13:54:37 -0500
Subject: [PATCH 34/37] Switch from fireEventually to `until`

---
 src/allmydata/immutable/upload.py | 102 +++++++++++++++++++++---------
 src/allmydata/test/test_upload.py |  27 ++++++++
 2 files changed, 98 insertions(+), 31 deletions(-)

diff --git a/src/allmydata/immutable/upload.py b/src/allmydata/immutable/upload.py
index adcdaed10..fe173b46c 100644
--- a/src/allmydata/immutable/upload.py
+++ b/src/allmydata/immutable/upload.py
@@ -13,19 +13,33 @@ if PY2:
 from past.builtins import long, unicode
 from six import ensure_str
 
+try:
+    from typing import List
+except ImportError:
+    pass
+
 import os, time, weakref, itertools
+from functools import (
+    partial,
+)
+
+import attr
+
 from zope.interface import implementer
 from twisted.python import failure
 from twisted.internet import defer
 from twisted.application import service
-from foolscap.api import Referenceable, Copyable, RemoteCopy, fireEventually
+from foolscap.api import Referenceable, Copyable, RemoteCopy
 
 from allmydata.crypto import aes
 from allmydata.util.hashutil import file_renewal_secret_hash, \
      file_cancel_secret_hash, bucket_renewal_secret_hash, \
      bucket_cancel_secret_hash, plaintext_hasher, \
      storage_index_hash, plaintext_segment_hasher, convergence_hasher
-from allmydata.util.deferredutil import timeout_call
+from allmydata.util.deferredutil import (
+    timeout_call,
+    until,
+)
 from allmydata import hashtree, uri
 from allmydata.storage.server import si_b2a
 from allmydata.immutable import encode
@@ -900,13 +914,41 @@ class Tahoe2ServerSelector(log.PrefixingLogMixin):
         raise UploadUnhappinessError(msg)
 
 
+@attr.s
+class _Accum(object):
+    """
+    Accumulate up to some known amount of ciphertext.
+
+    :ivar remaining: The number of bytes still expected.
+    :ivar ciphertext: The bytes accumulated so far.
+    """
+    remaining = attr.ib(validator=attr.validators.instance_of(int)) # type: int
+    ciphertext = attr.ib(default=attr.Factory(list))                # type: List[bytes]
+
+    def extend(self,
+               size,           # type: int
+               ciphertext,     # type: List[bytes]
+    ):
+        """
+        Accumulate some more ciphertext.
+
+        :param size: The amount of data the new ciphertext represents towards
+            the goal.  This may be more than the actual size of the given
+            ciphertext if the source has run out of data.
+
+        :param ciphertext: The new ciphertext to accumulate.
+        """
+        self.remaining -= size
+        self.ciphertext.extend(ciphertext)
+
+
 @implementer(IEncryptedUploadable)
 class EncryptAnUploadable(object):
     """This is a wrapper that takes an IUploadable and provides
     IEncryptedUploadable."""
     CHUNKSIZE = 50*1024
 
-    def __init__(self, original, log_parent=None, progress=None):
+    def __init__(self, original, log_parent=None, progress=None, chunk_size=None):
         precondition(original.default_params_set,
                      "set_default_encoding_parameters not called on %r before wrapping with EncryptAnUploadable" % (original,))
         self.original = IUploadable(original)
@@ -920,6 +962,8 @@ class EncryptAnUploadable(object):
         self._ciphertext_bytes_read = 0
         self._status = None
         self._progress = progress
+        if chunk_size is not None:
+            self.CHUNKSIZE = chunk_size
 
     def set_upload_status(self, upload_status):
         self._status = IUploadStatus(upload_status)
@@ -1026,47 +1070,43 @@ class EncryptAnUploadable(object):
         # and size
         d.addCallback(lambda ignored: self.get_size())
         d.addCallback(lambda ignored: self._get_encryptor())
-        # then fetch and encrypt the plaintext. The unusual structure here
-        # (passing a Deferred *into* a function) is needed to avoid
-        # overflowing the stack: Deferreds don't optimize out tail recursion.
-        # We also pass in a list, to which _read_encrypted will append
-        # ciphertext.
-        ciphertext = []
-        d2 = defer.Deferred()
-        d.addCallback(lambda ignored:
-                      self._read_encrypted(length, ciphertext, hash_only, d2))
-        d.addCallback(lambda ignored: d2)
+
+        accum = _Accum(length)
+        action = partial(self._read_encrypted, accum, hash_only)
+        condition = lambda: accum.remaining == 0
+
+        d.addCallback(lambda ignored: until(action, condition))
+        d.addCallback(lambda ignored: accum.ciphertext)
         return d
 
-    def _read_encrypted(self, remaining, ciphertext, hash_only, fire_when_done):
-        if not remaining:
-            fire_when_done.callback(ciphertext)
-            return None
+    def _read_encrypted(self,
+                        ciphertext_accum,  # type: _Accum
+                        hash_only,         # type: bool
+    ):
+        # type: (...) -> defer.Deferred
+        """
+        Read the next chunk of plaintext, encrypt it, and extend the accumulator
+        with the resulting ciphertext.
+        """
         # tolerate large length= values without consuming a lot of RAM by
         # reading just a chunk (say 50kB) at a time. This only really matters
         # when hash_only==True (i.e. resuming an interrupted upload), since
         # that's the case where we will be skipping over a lot of data.
-        size = min(remaining, self.CHUNKSIZE)
-        remaining = remaining - size
+        size = min(ciphertext_accum.remaining, self.CHUNKSIZE)
+
         # read a chunk of plaintext..
         d = defer.maybeDeferred(self.original.read, size)
-        # N.B.: if read() is synchronous, then since everything else is
-        # actually synchronous too, we'd blow the stack unless we stall for a
-        # tick. Once you accept a Deferred from IUploadable.read(), you must
-        # be prepared to have it fire immediately too.
-        d.addCallback(fireEventually)
         def _good(plaintext):
             # and encrypt it..
             # o/' over the fields we go, hashing all the way, sHA! sHA! sHA! o/'
             ct = self._hash_and_encrypt_plaintext(plaintext, hash_only)
-            ciphertext.extend(ct)
-            self._read_encrypted(remaining, ciphertext, hash_only,
-                                 fire_when_done)
-        def _err(why):
-            fire_when_done.errback(why)
+            # Intentionally tell the accumulator about the expected size, not
+            # the actual size.  If we run out of data we still want remaining
+            # to drop otherwise it will never reach 0 and the loop will never
+            # end.
+            ciphertext_accum.extend(size, ct)
         d.addCallback(_good)
-        d.addErrback(_err)
-        return None
+        return d
 
     def _hash_and_encrypt_plaintext(self, data, hash_only):
         assert isinstance(data, (tuple, list)), type(data)
diff --git a/src/allmydata/test/test_upload.py b/src/allmydata/test/test_upload.py
index 7e41bfc24..07ede2074 100644
--- a/src/allmydata/test/test_upload.py
+++ b/src/allmydata/test/test_upload.py
@@ -2097,6 +2097,33 @@ class EncryptAnUploadableTests(unittest.TestCase):
             b64encode(ciphertext),
         )
 
+    def test_large_read(self):
+        """
+        ``EncryptAnUploadable.read_encrypted`` succeeds even when the requested
+        data length is much larger than the chunk size.
+        """
+        convergence = b"\x42" * 16
+        # 4kB of plaintext
+        plaintext = b"\xde\xad\xbe\xef" * 1024
+        uploadable = upload.FileHandle(BytesIO(plaintext), convergence)
+        uploadable.set_default_encoding_parameters({
+            "k": 3,
+            "happy": 5,
+            "n": 10,
+            "max_segment_size": 128 * 1024,
+        })
+        # Make the chunk size very small so we don't have to operate on a huge
+        # amount of data to exercise the relevant codepath.
+        encrypter = upload.EncryptAnUploadable(uploadable, chunk_size=1)
+        d = encrypter.read_encrypted(len(plaintext), False)
+        ciphertext = self.successResultOf(d)
+        self.assertEqual(
+            list(map(len, ciphertext)),
+            # Chunk size was specified as 1 above so we will get the whole
+            # plaintext in one byte chunks.
+            [1] * len(plaintext),
+        )
+
 
 # TODO:
 #  upload with exactly 75 servers (shares_of_happiness)

From 9c91261fa6675e2f92890c9cd1229474e49883db Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Wed, 20 Jan 2021 13:57:01 -0500
Subject: [PATCH 35/37] news fragment

---
 newsfragments/3595.minor | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 create mode 100644 newsfragments/3595.minor

diff --git a/newsfragments/3595.minor b/newsfragments/3595.minor
new file mode 100644
index 000000000..e69de29bb

From 5a0c913f589de968e7543ba7175386454b509be3 Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Mon, 25 Jan 2021 08:21:39 -0500
Subject: [PATCH 36/37] document the new parameter

---
 src/allmydata/immutable/upload.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/allmydata/immutable/upload.py b/src/allmydata/immutable/upload.py
index fe173b46c..27cc923fd 100644
--- a/src/allmydata/immutable/upload.py
+++ b/src/allmydata/immutable/upload.py
@@ -949,6 +949,10 @@ class EncryptAnUploadable(object):
     CHUNKSIZE = 50*1024
 
     def __init__(self, original, log_parent=None, progress=None, chunk_size=None):
+        """
+        :param chunk_size: The number of bytes to read from the uploadable at a
+            time, or None for some default.
+        """
         precondition(original.default_params_set,
                      "set_default_encoding_parameters not called on %r before wrapping with EncryptAnUploadable" % (original,))
         self.original = IUploadable(original)

From e0fa2286228a46cd81b82868b56cf096d18d95dc Mon Sep 17 00:00:00 2001
From: Jean-Paul Calderone <exarkun@twistedmatrix.com>
Date: Mon, 25 Jan 2021 08:23:40 -0500
Subject: [PATCH 37/37] expand partial/lambda into full functions for clarity

---
 src/allmydata/immutable/upload.py | 17 ++++++++++++-----
 1 file changed, 12 insertions(+), 5 deletions(-)

diff --git a/src/allmydata/immutable/upload.py b/src/allmydata/immutable/upload.py
index 27cc923fd..46e01184f 100644
--- a/src/allmydata/immutable/upload.py
+++ b/src/allmydata/immutable/upload.py
@@ -19,9 +19,6 @@ except ImportError:
     pass
 
 import os, time, weakref, itertools
-from functools import (
-    partial,
-)
 
 import attr
 
@@ -1076,8 +1073,18 @@ class EncryptAnUploadable(object):
         d.addCallback(lambda ignored: self._get_encryptor())
 
         accum = _Accum(length)
-        action = partial(self._read_encrypted, accum, hash_only)
-        condition = lambda: accum.remaining == 0
+
+        def action():
+            """
+            Read some bytes into the accumulator.
+            """
+            return self._read_encrypted(accum, hash_only)
+
+        def condition():
+            """
+            Check to see if the accumulator has all the data.
+            """
+            return accum.remaining == 0
 
         d.addCallback(lambda ignored: until(action, condition))
         d.addCallback(lambda ignored: accum.ciphertext)