Merge pull request #966 from tahoe-lafs/3589.more-web-tests-python-3

Port more web tests to Python 3

Fixes ticket:3589

src/allmydata/test/test_websocket_logs.py

@@ -1,54 +0,0 @@
-import json
-
-from twisted.trial import unittest
-from twisted.internet.defer import inlineCallbacks
-
-from eliot import log_call
-
-from autobahn.twisted.testing import create_memory_agent, MemoryReactorClockResolver, create_pumper
-
-from allmydata.web.logs import TokenAuthenticatedWebSocketServerProtocol
-
-
-class TestStreamingLogs(unittest.TestCase):
-    """
-    Test websocket streaming of logs
-    """
-
-    def setUp(self):
-        self.reactor = MemoryReactorClockResolver()
-        self.pumper = create_pumper()
-        self.agent = create_memory_agent(self.reactor, self.pumper, TokenAuthenticatedWebSocketServerProtocol)
-        return self.pumper.start()
-
-    def tearDown(self):
-        return self.pumper.stop()
-
-    @inlineCallbacks
-    def test_one_log(self):
-        """
-        write a single Eliot log and see it streamed via websocket
-        """
-
-        proto = yield self.agent.open(
-            transport_config=u"ws://localhost:1234/ws",
-            options={},
-        )
-
-        messages = []
-        def got_message(msg, is_binary=False):
-            messages.append(json.loads(msg))
-        proto.on("message", got_message)
-
-        @log_call(action_type=u"test:cli:some-exciting-action")
-        def do_a_thing():
-            pass
-
-        do_a_thing()
-
-        proto.transport.loseConnection()
-        yield proto.is_closed
-
-        self.assertEqual(len(messages), 2)
-        self.assertEqual("started", messages[0]["action_status"])
-        self.assertEqual("succeeded", messages[1]["action_status"])

src/allmydata/test/web/test_introducer.py

@@ -1,3 +1,15 @@
+"""
+Ported to Python 3.
+"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from future.utils import PY2
+if PY2:
+    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
+
 import json
 from os.path import join
 
@@ -213,7 +225,7 @@ class IntroducerRootTests(unittest.TestCase):
         resource = IntroducerRoot(introducer_node)
         response = json.loads(
             self.successResultOf(
-                render(resource, {"t": [b"json"]}),
+                render(resource, {b"t": [b"json"]}),
             ),
         )
         self.assertEqual(

src/allmydata/test/web/test_logs.py

@@ -1,5 +1,7 @@
 """
 Tests for ``allmydata.web.logs``.
+
+Ported to Python 3.
 """
 
 from __future__ import (
@@ -9,6 +11,19 @@ from __future__ import (
     division,
 )
 
+from future.utils import PY2
+if PY2:
+    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
+
+import json
+
+from twisted.trial import unittest
+from twisted.internet.defer import inlineCallbacks
+
+from eliot import log_call
+
+from autobahn.twisted.testing import create_memory_agent, MemoryReactorClockResolver, create_pumper
+
 from testtools.matchers import (
     Equals,
 )
@@ -37,6 +52,7 @@ from ..common import (
 
 from ...web.logs import (
     create_log_resources,
+    TokenAuthenticatedWebSocketServerProtocol,
 )
 
 class StreamingEliotLogsTests(SyncTestCase):
@@ -57,3 +73,47 @@ class StreamingEliotLogsTests(SyncTestCase):
             self.client.get(b"http:///v1"),
             succeeded(has_response_code(Equals(OK))),
         )
+
+
+class TestStreamingLogs(unittest.TestCase):
+    """
+    Test websocket streaming of logs
+    """
+
+    def setUp(self):
+        self.reactor = MemoryReactorClockResolver()
+        self.pumper = create_pumper()
+        self.agent = create_memory_agent(self.reactor, self.pumper, TokenAuthenticatedWebSocketServerProtocol)
+        return self.pumper.start()
+
+    def tearDown(self):
+        return self.pumper.stop()
+
+    @inlineCallbacks
+    def test_one_log(self):
+        """
+        write a single Eliot log and see it streamed via websocket
+        """
+
+        proto = yield self.agent.open(
+            transport_config=u"ws://localhost:1234/ws",
+            options={},
+        )
+
+        messages = []
+        def got_message(msg, is_binary=False):
+            messages.append(json.loads(msg))
+        proto.on("message", got_message)
+
+        @log_call(action_type=u"test:cli:some-exciting-action")
+        def do_a_thing():
+            pass
+
+        do_a_thing()
+
+        proto.transport.loseConnection()
+        yield proto.is_closed
+
+        self.assertEqual(len(messages), 2)
+        self.assertEqual("started", messages[0]["action_status"])
+        self.assertEqual("succeeded", messages[1]["action_status"])

src/allmydata/test/web/test_private.py

@@ -1,5 +1,7 @@
 """
 Tests for ``allmydata.web.private``.
+
+Ported to Python 3.
 """
 
 from __future__ import (
@@ -9,6 +11,10 @@ from __future__ import (
     division,
 )
 
+from future.utils import PY2
+if PY2:
+    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
+
 from testtools.matchers import (
     Equals,
 )
@@ -56,6 +62,7 @@ class PrivacyTests(SyncTestCase):
         return super(PrivacyTests, self).setUp()
 
     def _authorization(self, scheme, value):
+        value = str(value, "utf-8")
         return Headers({
             u"authorization": [u"{} {}".format(scheme, value)],
         })
@@ -90,7 +97,7 @@ class PrivacyTests(SyncTestCase):
         self.assertThat(
             self.client.head(
                 b"http:///foo/bar",
-                headers=self._authorization(SCHEME, u"foo bar"),
+                headers=self._authorization(str(SCHEME, "utf-8"), b"foo bar"),
             ),
             succeeded(has_response_code(Equals(UNAUTHORIZED))),
         )
@@ -103,7 +110,7 @@ class PrivacyTests(SyncTestCase):
         self.assertThat(
             self.client.head(
                 b"http:///foo/bar",
-                headers=self._authorization(SCHEME, self.token),
+                headers=self._authorization(str(SCHEME, "utf-8"), self.token),
             ),
             # It's a made up URL so we don't get a 200, either, but a 404.
             succeeded(has_response_code(Equals(NOT_FOUND))),

src/allmydata/test/web/test_root.py

@@ -1,7 +1,20 @@
+"""
+Ported to Python 3.
+"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from future.utils import PY2
+if PY2:
+    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
+
 import time
 
-
+from urllib.parse import (
-from six.moves.urllib.parse import quote
+    quote,
+)
 
 from bs4 import (
     BeautifulSoup,
@@ -76,7 +89,7 @@ class RenderSlashUri(unittest.TestCase):
         )
         self.assertEqual(
             response_body,
-            "Invalid capability",
+            b"Invalid capability",
         )
 
 
@@ -91,7 +104,7 @@ class RenderServiceRow(unittest.TestCase):
         ann = {"anonymous-storage-FURL": "pb://w2hqnbaa25yw4qgcvghl5psa3srpfgw3@tcp:127.0.0.1:51309/vucto2z4fxment3vfxbqecblbf6zyp6x",
                "permutation-seed-base32": "w2hqnbaa25yw4qgcvghl5psa3srpfgw3",
                }
-        srv = NativeStorageServer("server_id", ann, None, {}, EMPTY_CLIENT_CONFIG)
+        srv = NativeStorageServer(b"server_id", ann, None, {}, EMPTY_CLIENT_CONFIG)
         srv.get_connection_status = lambda: ConnectionStatus(False, "summary", {}, 0, 0)
 
         class FakeClient(_Client):
@@ -102,7 +115,7 @@ class RenderServiceRow(unittest.TestCase):
                     tub_maker=None,
                     node_config=EMPTY_CLIENT_CONFIG,
                 )
-                self.storage_broker.test_add_server("test-srv", srv)
+                self.storage_broker.test_add_server(b"test-srv", srv)
 
         root = RootElement(FakeClient(), time.time)
         req = DummyRequest(b"")

src/allmydata/util/_python3.py

@@ -114,6 +114,7 @@ PORTED_MODULES = [
     "allmydata.util.spans",
     "allmydata.util.statistics",
     "allmydata.util.time_format",
+    "allmydata.web.logs",
     "allmydata.webish",
 ]
 
@@ -187,6 +188,10 @@ PORTED_TEST_MODULES = [
     "allmydata.test.test_util",
     "allmydata.test.web.test_common",
     "allmydata.test.web.test_grid",
+    "allmydata.test.web.test_introducer",
+    "allmydata.test.web.test_logs",
+    "allmydata.test.web.test_private",
+    "allmydata.test.web.test_root",
     "allmydata.test.web.test_status",
     "allmydata.test.web.test_util",
     "allmydata.test.web.test_webish",

src/allmydata/web/introweb.py

@@ -26,10 +26,10 @@ class IntroducerRoot(MultiFormatResource):
         self.introducer_node = introducer_node
         self.introducer_service = introducer_node.getServiceNamed("introducer")
         # necessary as a root Resource
-        self.putChild("", self)
+        self.putChild(b"", self)
         static_dir = resource_filename("allmydata.web", "static")
         for filen in os.listdir(static_dir):
-            self.putChild(filen, static.File(os.path.join(static_dir, filen)))
+            self.putChild(filen.encode("utf-8"), static.File(os.path.join(static_dir, filen)))
 
     def _create_element(self):
         """
@@ -66,7 +66,7 @@ class IntroducerRoot(MultiFormatResource):
             announcement_summary[service_name] += 1
         res[u"announcement_summary"] = announcement_summary
 
-        return json.dumps(res, indent=1) + b"\n"
+        return (json.dumps(res, indent=1) + "\n").encode("utf-8")
 
 
 class IntroducerRootElement(Element):

src/allmydata/web/logs.py

@@ -1,3 +1,6 @@
+"""
+Ported to Python 3.
+"""
 from __future__ import (
     print_function,
     unicode_literals,
@@ -49,7 +52,11 @@ class TokenAuthenticatedWebSocketServerProtocol(WebSocketServerProtocol):
         """
         # probably want a try/except around here? what do we do if
         # transmission fails or anything else bad happens?
-        self.sendMessage(json.dumps(message))
+        encoded = json.dumps(message)
+        if isinstance(encoded, str):
+            # On Python 3 dumps() returns Unicode...
+            encoded = encoded.encode("utf-8")
+        self.sendMessage(encoded)
 
     def onOpen(self):
         """

src/allmydata/web/root.py

@@ -1,6 +1,8 @@
+from future.utils import PY3
+
 import os
 import time
-import urllib
+from urllib.parse import quote as urlquote
 
 from hyperlink import DecodedURL, URL
 from pkg_resources import resource_filename
@@ -81,7 +83,7 @@ class URIHandler(resource.Resource, object):
         # it seems Nevow was creating absolute URLs including
         # host/port whereas req.uri is absolute (but lacks host/port)
         redir_uri = URL.from_text(req.prePathURL().decode('utf8'))
-        redir_uri = redir_uri.child(urllib.quote(uri_arg).decode('utf8'))
+        redir_uri = redir_uri.child(urlquote(uri_arg))
         # add back all the query args that AREN'T "?uri="
         for k, values in req.args.items():
             if k != b"uri":
@@ -227,26 +229,26 @@ class Root(MultiFormatResource):
         self._client = client
         self._now_fn = now_fn
 
-        # Children need to be bytes; for now just doing these to make specific
-        # tests pass on Python 3, but eventually will do all them when this
-        # module is ported to Python 3 (if not earlier).
         self.putChild(b"uri", URIHandler(client))
-        self.putChild("cap", URIHandler(client))
+        self.putChild(b"cap", URIHandler(client))
 
         # Handler for everything beneath "/private", an area of the resource
         # hierarchy which is only accessible with the private per-node API
         # auth token.
-        self.putChild("private", create_private_tree(client.get_auth_token))
+        self.putChild(b"private", create_private_tree(client.get_auth_token))
 
-        self.putChild("file", FileHandler(client))
+        self.putChild(b"file", FileHandler(client))
-        self.putChild("named", FileHandler(client))
+        self.putChild(b"named", FileHandler(client))
-        self.putChild("status", status.Status(client.get_history()))
+        self.putChild(b"status", status.Status(client.get_history()))
-        self.putChild("statistics", status.Statistics(client.stats_provider))
+        self.putChild(b"statistics", status.Statistics(client.stats_provider))
         static_dir = resource_filename("allmydata.web", "static")
         for filen in os.listdir(static_dir):
-            self.putChild(filen, static.File(os.path.join(static_dir, filen)))
+            child_path = filen
+            if PY3:
+                child_path = filen.encode("utf-8")
+            self.putChild(child_path, static.File(os.path.join(static_dir, filen)))
 
-        self.putChild("report_incident", IncidentReporter())
+        self.putChild(b"report_incident", IncidentReporter())
 
     @exception_to_child
     def getChild(self, path, request):