Support broader range of server endpoints, and switch to more robust random port

assignment.
This commit is contained in:
Itamar Turner-Trauring 2022-04-06 11:10:42 -04:00
parent 2e934574f0
commit 710fad4f8a
2 changed files with 42 additions and 39 deletions

View File

@ -2,19 +2,21 @@
HTTP server for storage. HTTP server for storage.
""" """
from typing import Dict, List, Set, Tuple, Any, Optional from typing import Dict, List, Set, Tuple, Any
from pathlib import Path from pathlib import Path
from functools import wraps from functools import wraps
from base64 import b64decode from base64 import b64decode
import binascii import binascii
from zope.interface import implementer
from klein import Klein from klein import Klein
from twisted.web import http from twisted.web import http
from twisted.internet.interfaces import IListeningPort from twisted.internet.interfaces import IListeningPort, IStreamServerEndpoint
from twisted.internet.defer import Deferred from twisted.internet.defer import Deferred
from twisted.internet.endpoints import quoteStringArgument, serverFromString from twisted.internet.ssl import CertificateOptions, Certificate, PrivateCertificate
from twisted.web.server import Site from twisted.web.server import Site
from twisted.protocols.tls import TLSMemoryBIOFactory
import attr import attr
from werkzeug.http import ( from werkzeug.http import (
parse_range_header, parse_range_header,
@ -518,33 +520,48 @@ class HTTPServer(object):
return b"" return b""
@implementer(IStreamServerEndpoint)
@attr.s
class _TLSEndpointWrapper(object):
"""
Wrap an existing endpoint with the storage TLS policy. This is useful
because not all Tahoe-LAFS endpoints might be plain TCP+TLS, for example
there's Tor and i2p.
"""
endpoint = attr.ib(type=IStreamServerEndpoint)
context_factory = attr.ib(type=CertificateOptions)
def listen(self, factory):
return self.endpoint.listen(
TLSMemoryBIOFactory(self.context_factory, False, factory)
)
def listen_tls( def listen_tls(
reactor,
server: HTTPServer, server: HTTPServer,
hostname: str, hostname: str,
port: int, endpoint: IStreamServerEndpoint,
private_key_path: Path, private_key_path: Path,
cert_path: Path, cert_path: Path,
interface: Optional[str],
) -> Deferred[Tuple[DecodedURL, IListeningPort]]: ) -> Deferred[Tuple[DecodedURL, IListeningPort]]:
""" """
Start a HTTPS storage server on the given port, return the NURL and the Start a HTTPS storage server on the given port, return the NURL and the
listening port. listening port.
The hostname is the external IP or hostname clients will connect to; it The hostname is the external IP or hostname clients will connect to, used
does not modify what interfaces the server listens on. To set the to constrtuct the NURL; it does not modify what interfaces the server
listening interface, use the ``interface`` argument. listens on.
Port can be 0 to choose a random port. This will likely need to be updated eventually to handle Tor/i2p.
""" """
endpoint_string = "ssl:privateKey={}:certKey={}:port={}".format( certificate = Certificate.loadPEM(cert_path.read_bytes()).original
quoteStringArgument(str(private_key_path)), private_key = PrivateCertificate.loadPEM(
quoteStringArgument(str(cert_path)), cert_path.read_bytes() + b"\n" + private_key_path.read_bytes()
port, ).privateKey.original
endpoint = _TLSEndpointWrapper(
endpoint, CertificateOptions(privateKey=private_key, certificate=certificate)
) )
if interface is not None:
endpoint_string += ":interface={}".format(quoteStringArgument(interface))
endpoint = serverFromString(reactor, endpoint_string)
def build_nurl(listening_port: IListeningPort) -> DecodedURL: def build_nurl(listening_port: IListeningPort) -> DecodedURL:
nurl = DecodedURL().replace( nurl = DecodedURL().replace(

View File

@ -8,19 +8,9 @@ reused across tests, so each test should be careful to generate unique storage
indexes. indexes.
""" """
from __future__ import absolute_import from future.utils import bchr
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
from future.utils import PY2, bchr from typing import Set
if PY2:
# fmt: off
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
# fmt: on
else:
from typing import Set
from random import Random from random import Random
from unittest import SkipTest from unittest import SkipTest
@ -29,7 +19,7 @@ from pathlib import Path
from twisted.internet.defer import inlineCallbacks, returnValue, succeed from twisted.internet.defer import inlineCallbacks, returnValue, succeed
from twisted.internet.task import Clock from twisted.internet.task import Clock
from twisted.internet import reactor from twisted.internet import reactor
from twisted.internet.endpoints import serverFromString
from foolscap.api import Referenceable, RemoteException from foolscap.api import Referenceable, RemoteException
from allmydata.interfaces import IStorageServer # really, IStorageClient from allmydata.interfaces import IStorageServer # really, IStorageClient
@ -1013,10 +1003,6 @@ class _SharedMixin(SystemTestMixin):
AsyncTestCase.setUp(self) AsyncTestCase.setUp(self)
self._port_assigner = SameProcessStreamEndpointAssigner()
self._port_assigner.setUp()
self.addCleanup(self._port_assigner.tearDown)
self.basedir = "test_istorageserver/" + self.id() self.basedir = "test_istorageserver/" + self.id()
yield SystemTestMixin.setUp(self) yield SystemTestMixin.setUp(self)
yield self.set_up_nodes(1) yield self.set_up_nodes(1)
@ -1061,8 +1047,9 @@ class _HTTPMixin(_SharedMixin):
"""Run tests on the HTTP version of ``IStorageServer``.""" """Run tests on the HTTP version of ``IStorageServer``."""
def setUp(self): def setUp(self):
if PY2: self._port_assigner = SameProcessStreamEndpointAssigner()
self.skipTest("Not going to bother supporting Python 2") self._port_assigner.setUp()
self.addCleanup(self._port_assigner.tearDown)
return _SharedMixin.setUp(self) return _SharedMixin.setUp(self)
@inlineCallbacks @inlineCallbacks
@ -1073,18 +1060,17 @@ class _HTTPMixin(_SharedMixin):
# Listen on randomly assigned port, using self-signed cert we generated # Listen on randomly assigned port, using self-signed cert we generated
# manually: # manually:
certs_dir = Path(__file__).parent / "certs" certs_dir = Path(__file__).parent / "certs"
_, endpoint_string = self._port_assigner.assign(reactor)
nurl, listening_port = yield listen_tls( nurl, listening_port = yield listen_tls(
reactor,
http_storage_server, http_storage_server,
"127.0.0.1", "127.0.0.1",
0, serverFromString(reactor, endpoint_string),
# This is just a self-signed certificate with randomly generated # This is just a self-signed certificate with randomly generated
# private key; nothing at all special about it. You can regenerate # private key; nothing at all special about it. You can regenerate
# with code in allmydata.test.test_storage_https or with openssl # with code in allmydata.test.test_storage_https or with openssl
# CLI, with no meaningful change to the test. # CLI, with no meaningful change to the test.
certs_dir / "private.key", certs_dir / "private.key",
certs_dir / "domain.crt", certs_dir / "domain.crt",
interface="127.0.0.1",
) )
self.addCleanup(listening_port.stopListening) self.addCleanup(listening_port.stopListening)