Support broader range of server endpoints, and switch to more robust random port

assignment.

src/allmydata/storage/http_server.py

@@ -2,19 +2,21 @@
 HTTP server for storage.
 """
 
-from typing import Dict, List, Set, Tuple, Any, Optional
+from typing import Dict, List, Set, Tuple, Any
 from pathlib import Path
 
 from functools import wraps
 from base64 import b64decode
 import binascii
 
+from zope.interface import implementer
 from klein import Klein
 from twisted.web import http
-from twisted.internet.interfaces import IListeningPort
+from twisted.internet.interfaces import IListeningPort, IStreamServerEndpoint
 from twisted.internet.defer import Deferred
-from twisted.internet.endpoints import quoteStringArgument, serverFromString
+from twisted.internet.ssl import CertificateOptions, Certificate, PrivateCertificate
 from twisted.web.server import Site
+from twisted.protocols.tls import TLSMemoryBIOFactory
 import attr
 from werkzeug.http import (
     parse_range_header,
@@ -518,33 +520,48 @@ class HTTPServer(object):
         return b""
 
 
+@implementer(IStreamServerEndpoint)
+@attr.s
+class _TLSEndpointWrapper(object):
+    """
+    Wrap an existing endpoint with the storage TLS policy.  This is useful
+    because not all Tahoe-LAFS endpoints might be plain TCP+TLS, for example
+    there's Tor and i2p.
+    """
+
+    endpoint = attr.ib(type=IStreamServerEndpoint)
+    context_factory = attr.ib(type=CertificateOptions)
+
+    def listen(self, factory):
+        return self.endpoint.listen(
+            TLSMemoryBIOFactory(self.context_factory, False, factory)
+        )
+
+
 def listen_tls(
-    reactor,
     server: HTTPServer,
     hostname: str,
-    port: int,
+    endpoint: IStreamServerEndpoint,
     private_key_path: Path,
     cert_path: Path,
-    interface: Optional[str],
 ) -> Deferred[Tuple[DecodedURL, IListeningPort]]:
     """
     Start a HTTPS storage server on the given port, return the NURL and the
     listening port.
 
-    The hostname is the external IP or hostname clients will connect to; it
-    does not modify what interfaces the server listens on.  To set the
-    listening interface, use the ``interface`` argument.
+    The hostname is the external IP or hostname clients will connect to, used
+    to constrtuct the NURL; it does not modify what interfaces the server
+    listens on.
 
-    Port can be 0 to choose a random port.
+    This will likely need to be updated eventually to handle Tor/i2p.
     """
-    endpoint_string = "ssl:privateKey={}:certKey={}:port={}".format(
-        quoteStringArgument(str(private_key_path)),
-        quoteStringArgument(str(cert_path)),
-        port,
+    certificate = Certificate.loadPEM(cert_path.read_bytes()).original
+    private_key = PrivateCertificate.loadPEM(
+        cert_path.read_bytes() + b"\n" + private_key_path.read_bytes()
+    ).privateKey.original
+    endpoint = _TLSEndpointWrapper(
+        endpoint, CertificateOptions(privateKey=private_key, certificate=certificate)
     )
-    if interface is not None:
-        endpoint_string += ":interface={}".format(quoteStringArgument(interface))
-    endpoint = serverFromString(reactor, endpoint_string)
 
     def build_nurl(listening_port: IListeningPort) -> DecodedURL:
         nurl = DecodedURL().replace(

src/allmydata/test/test_istorageserver.py

@@ -8,18 +8,8 @@ reused across tests, so each test should be careful to generate unique storage
 indexes.
 """
 
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-from __future__ import unicode_literals
+from future.utils import bchr
 
-from future.utils import PY2, bchr
-
-if PY2:
-    # fmt: off
-    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
-    # fmt: on
-else:
 from typing import Set
 
 from random import Random
@@ -29,7 +19,7 @@ from pathlib import Path
 from twisted.internet.defer import inlineCallbacks, returnValue, succeed
 from twisted.internet.task import Clock
 from twisted.internet import reactor
-
+from twisted.internet.endpoints import serverFromString
 from foolscap.api import Referenceable, RemoteException
 
 from allmydata.interfaces import IStorageServer  # really, IStorageClient
@@ -1013,10 +1003,6 @@ class _SharedMixin(SystemTestMixin):
 
         AsyncTestCase.setUp(self)
 
-        self._port_assigner = SameProcessStreamEndpointAssigner()
-        self._port_assigner.setUp()
-        self.addCleanup(self._port_assigner.tearDown)
-
         self.basedir = "test_istorageserver/" + self.id()
         yield SystemTestMixin.setUp(self)
         yield self.set_up_nodes(1)
@@ -1061,8 +1047,9 @@ class _HTTPMixin(_SharedMixin):
     """Run tests on the HTTP version of ``IStorageServer``."""
 
     def setUp(self):
-        if PY2:
-            self.skipTest("Not going to bother supporting Python 2")
+        self._port_assigner = SameProcessStreamEndpointAssigner()
+        self._port_assigner.setUp()
+        self.addCleanup(self._port_assigner.tearDown)
         return _SharedMixin.setUp(self)
 
     @inlineCallbacks
@@ -1073,18 +1060,17 @@ class _HTTPMixin(_SharedMixin):
         # Listen on randomly assigned port, using self-signed cert we generated
         # manually:
         certs_dir = Path(__file__).parent / "certs"
+        _, endpoint_string = self._port_assigner.assign(reactor)
         nurl, listening_port = yield listen_tls(
-            reactor,
             http_storage_server,
             "127.0.0.1",
-            0,
+            serverFromString(reactor, endpoint_string),
             # This is just a self-signed certificate with randomly generated
             # private key; nothing at all special about it. You can regenerate
             # with code in allmydata.test.test_storage_https or with openssl
             # CLI, with no meaningful change to the test.
             certs_dir / "private.key",
             certs_dir / "domain.crt",
-            interface="127.0.0.1",
         )
         self.addCleanup(listening_port.stopListening)