Merge branch '3843-start-http-storage-server' into 3849-refactor-out-foolscap-in-storage-server

nix/cbor2.nix

@@ -0,0 +1,20 @@
+{ lib, buildPythonPackage, fetchPypi, setuptools_scm }:
+buildPythonPackage rec {
+  pname = "cbor2";
+  version = "5.2.0";
+
+  src = fetchPypi {
+    sha256 = "1gwlgjl70vlv35cgkcw3cg7b5qsmws36hs4mmh0l9msgagjs4fm3";
+    inherit pname version;
+  };
+
+  doCheck = false;
+
+  propagatedBuildInputs = [ setuptools_scm ];
+
+  meta = with lib; {
+    homepage = https://github.com/agronholm/cbor2;
+    description = "CBOR encoder/decoder";
+    license = licenses.mit;
+  };
+}

nix/overlays.nix

@@ -21,6 +21,9 @@ self: super: {
 
       # collections-extended is not part of nixpkgs at this time.
       collections-extended = python-super.pythonPackages.callPackage ./collections-extended.nix { };
+
+      # cbor2 is not part of nixpkgs at this time.
+      cbor2 = python-super.pythonPackages.callPackage ./cbor2.nix { };
     };
   };
 

nix/tahoe-lafs.nix

@@ -4,7 +4,7 @@
 , setuptools, setuptoolsTrial, pyasn1, zope_interface
 , service-identity, pyyaml, magic-wormhole, treq, appdirs
 , beautifulsoup4, eliot, autobahn, cryptography, netifaces
-, html5lib, pyutil, distro, configparser
+, html5lib, pyutil, distro, configparser, klein, cbor2
 }:
 python.pkgs.buildPythonPackage rec {
   # Most of the time this is not exactly the release version (eg 1.16.0).
@@ -95,9 +95,10 @@ EOF
   propagatedBuildInputs = with python.pkgs; [
     twisted foolscap zfec appdirs
     setuptoolsTrial pyasn1 zope_interface
-    service-identity pyyaml magic-wormhole treq
+    service-identity pyyaml magic-wormhole
     eliot autobahn cryptography netifaces setuptools
     future pyutil distro configparser collections-extended
+    klein cbor2 treq
   ];
 
   checkInputs = with python.pkgs; [

setup.py

@@ -140,6 +140,11 @@ install_requires = [
 
     # For the RangeMap datastructure.
     "collections-extended",
+
+    # HTTP server and client
+    "klein",
+    "treq",
+    "cbor2"
 ]
 
 setup_requires = [
@@ -397,7 +402,6 @@ setup(name="tahoe-lafs", # also set in __init__.py
               # Python 2.7.
               "decorator < 5",
               "hypothesis >= 3.6.1",
-              "treq",
               "towncrier",
               "testtools",
               "fixtures",

src/allmydata/storage/http_client.py

@@ -0,0 +1,79 @@
+"""
+HTTP client that talks to the HTTP storage server.
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from future.utils import PY2
+
+if PY2:
+    # fmt: off
+    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
+    # fmt: on
+else:
+    # typing module not available in Python 2, and we only do type checking in
+    # Python 3 anyway.
+    from typing import Union
+    from treq.testing import StubTreq
+
+import base64
+
+# TODO Make sure to import Python version?
+from cbor2 import loads
+
+
+from twisted.web.http_headers import Headers
+from twisted.internet.defer import inlineCallbacks, returnValue, fail
+from hyperlink import DecodedURL
+import treq
+
+
+class ClientException(Exception):
+    """An unexpected error."""
+
+
+def _decode_cbor(response):
+    """Given HTTP response, return decoded CBOR body."""
+    if response.code > 199 and response.code < 300:
+        return treq.content(response).addCallback(loads)
+    return fail(ClientException(response.code, response.phrase))
+
+
+def swissnum_auth_header(swissnum):  # type: (bytes) -> bytes
+    """Return value for ``Authentication`` header."""
+    return b"Tahoe-LAFS " + base64.b64encode(swissnum).strip()
+
+
+class StorageClient(object):
+    """
+    HTTP client that talks to the HTTP storage server.
+    """
+
+    def __init__(
+        self, url, swissnum, treq=treq
+    ):  # type: (DecodedURL, bytes, Union[treq,StubTreq]) -> None
+        self._base_url = url
+        self._swissnum = swissnum
+        self._treq = treq
+
+    def _get_headers(self):  # type: () -> Headers
+        """Return the basic headers to be used by default."""
+        headers = Headers()
+        headers.addRawHeader(
+            "Authorization",
+            swissnum_auth_header(self._swissnum),
+        )
+        return headers
+
+    @inlineCallbacks
+    def get_version(self):
+        """
+        Return the version metadata for the server.
+        """
+        url = self._base_url.click("/v1/version")
+        response = yield self._treq.get(url, headers=self._get_headers())
+        decoded_response = yield _decode_cbor(response)
+        returnValue(decoded_response)

src/allmydata/storage/http_server.py

@@ -0,0 +1,94 @@
+"""
+HTTP server for storage.
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from future.utils import PY2
+
+if PY2:
+    # fmt: off
+    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
+    # fmt: on
+
+from functools import wraps
+
+from klein import Klein
+from twisted.web import http
+
+# TODO Make sure to use pure Python versions?
+from cbor2 import dumps
+
+from .server import StorageServer
+from .http_client import swissnum_auth_header
+
+
+def _authorization_decorator(f):
+    """
+    Check the ``Authorization`` header, and (TODO: in later revision of code)
+    extract ``X-Tahoe-Authorization`` headers and pass them in.
+    """
+
+    @wraps(f)
+    def route(self, request, *args, **kwargs):
+        if request.requestHeaders.getRawHeaders("Authorization", [None])[0] != str(
+            swissnum_auth_header(self._swissnum), "ascii"
+        ):
+            request.setResponseCode(http.UNAUTHORIZED)
+            return b""
+        # authorization = request.requestHeaders.getRawHeaders("X-Tahoe-Authorization", [])
+        # For now, just a placeholder:
+        authorization = None
+        return f(self, request, authorization, *args, **kwargs)
+
+    return route
+
+
+def _authorized_route(app, *route_args, **route_kwargs):
+    """
+    Like Klein's @route, but with additional support for checking the
+    ``Authorization`` header as well as ``X-Tahoe-Authorization`` headers.  The
+    latter will (TODO: in later revision of code) get passed in as second
+    argument to wrapped functions.
+    """
+
+    def decorator(f):
+        @app.route(*route_args, **route_kwargs)
+        @_authorization_decorator
+        def handle_route(*args, **kwargs):
+            return f(*args, **kwargs)
+
+        return handle_route
+
+    return decorator
+
+
+class HTTPServer(object):
+    """
+    A HTTP interface to the storage server.
+    """
+
+    _app = Klein()
+
+    def __init__(
+        self, storage_server, swissnum
+    ):  # type: (StorageServer, bytes) -> None
+        self._storage_server = storage_server
+        self._swissnum = swissnum
+
+    def get_resource(self):
+        """Return twisted.web ``Resource`` for this object."""
+        return self._app.resource()
+
+    def _cbor(self, request, data):
+        """Return CBOR-encoded data."""
+        request.setHeader("Content-Type", "application/cbor")
+        # TODO if data is big, maybe want to use a temporary file eventually...
+        return dumps(data)
+
+    @_authorized_route(_app, "/v1/version", methods=["GET"])
+    def version(self, request, authorization):
+        return self._cbor(request, self._storage_server.remote_get_version())

src/allmydata/test/test_storage_http.py

@@ -0,0 +1,69 @@
+"""
+Tests for HTTP storage client + server.
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from future.utils import PY2
+
+if PY2:
+    # fmt: off
+    from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min  # noqa: F401
+    # fmt: on
+
+from unittest import SkipTest
+
+from twisted.trial.unittest import TestCase
+from twisted.internet.defer import inlineCallbacks
+
+from treq.testing import StubTreq
+from hyperlink import DecodedURL
+
+from ..storage.server import StorageServer
+from ..storage.http_server import HTTPServer
+from ..storage.http_client import StorageClient, ClientException
+
+
+class HTTPTests(TestCase):
+    """
+    Tests of HTTP client talking to the HTTP server.
+    """
+
+    def setUp(self):
+        if PY2:
+            raise SkipTest("Not going to bother supporting Python 2")
+        self.storage_server = StorageServer(self.mktemp(), b"\x00" * 20)
+        # TODO what should the swissnum _actually_ be?
+        self._http_server = HTTPServer(self.storage_server, b"abcd")
+        self.client = StorageClient(
+            DecodedURL.from_text("http://127.0.0.1"),
+            b"abcd",
+            treq=StubTreq(self._http_server.get_resource()),
+        )
+
+    @inlineCallbacks
+    def test_bad_authentication(self):
+        """
+        If the wrong swissnum is used, an ``Unauthorized`` response code is
+        returned.
+        """
+        client = StorageClient(
+            DecodedURL.from_text("http://127.0.0.1"),
+            b"something wrong",
+            treq=StubTreq(self._http_server.get_resource()),
+        )
+        with self.assertRaises(ClientException) as e:
+            yield client.get_version()
+        self.assertEqual(e.exception.args[0], 401)
+
+    @inlineCallbacks
+    def test_version(self):
+        """
+        The client can return the version.
+        """
+        version = yield self.client.get_version()
+        expected_version = self.storage_server.remote_get_version()
+        self.assertEqual(version, expected_version)