From 4ea005d653b1ba299e2a966a8c159cf380a90e48 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 6 Jul 2023 13:18:18 -0400 Subject: [PATCH] A unit test, and some cleanups and fixes. --- src/allmydata/client.py | 3 ++- src/allmydata/storage/http_common.py | 4 ++++ src/allmydata/storage/server.py | 9 +++++---- src/allmydata/storage_client.py | 5 +++-- src/allmydata/test/test_storage.py | 14 ++++++++++++++ 5 files changed, 28 insertions(+), 7 deletions(-) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index 03316a45c..b8fe464a6 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -60,6 +60,7 @@ from allmydata.interfaces import ( from allmydata.nodemaker import NodeMaker from allmydata.blacklist import Blacklist from allmydata.node import _Config +from allmydata.storage.http_common import ANONYMOUS_STORAGE_NURLS KiB=1024 MiB=1024*KiB @@ -838,7 +839,7 @@ class _Client(node.Node, pollmixin.PollMixin): if hasattr(self.tub.negotiationClass, "add_storage_server"): nurls = self.tub.negotiationClass.add_storage_server(ss, swissnum.encode("ascii")) self.storage_nurls = nurls - announcement[storage_client.ANONYMOUS_STORAGE_NURLS] = [n.to_text() for n in nurls] + announcement[ANONYMOUS_STORAGE_NURLS] = [n.to_text() for n in nurls] # Tell Foolscap server about the nurls so it can send them out # to clients, allowing them to auto-upgrade. foolscap_server.set_nurls(nurls) diff --git a/src/allmydata/storage/http_common.py b/src/allmydata/storage/http_common.py index 650d905e9..e3721e1e4 100644 --- a/src/allmydata/storage/http_common.py +++ b/src/allmydata/storage/http_common.py @@ -16,6 +16,10 @@ from twisted.web.iweb import IResponse CBOR_MIME_TYPE = "application/cbor" +# Key used in introducer announcements and Foolscap storage server version for +# HTTP storage NURLs: +ANONYMOUS_STORAGE_NURLS = "anonymous-storage-NURLs" + def get_content_type(headers: Headers) -> Optional[str]: """ diff --git a/src/allmydata/storage/server.py b/src/allmydata/storage/server.py index 0bd26004d..ef522e7ad 100644 --- a/src/allmydata/storage/server.py +++ b/src/allmydata/storage/server.py @@ -31,7 +31,7 @@ from allmydata.storage.immutable import ( ) from allmydata.storage.crawler import BucketCountingCrawler from allmydata.storage.expirer import LeaseCheckingCrawler -from ..storage_client import ANONYMOUS_STORAGE_NURLS +from .http_common import ANONYMOUS_STORAGE_NURLS # storage/ # storage/shares/incoming @@ -834,7 +834,7 @@ class FoolscapStorageServer(Referenceable): # type: ignore # warner/foolscap#78 # The HTTP storage NURLs for this server. We include them in version # messages so clients can upgrade from Foolscap to HTTP. - self._nurls : list[str] = [] + self._nurls : list[bytes] = [] def _bucket_writer_closed(self, bw): if bw in self._bucket_writer_disconnect_markers: @@ -843,13 +843,14 @@ class FoolscapStorageServer(Referenceable): # type: ignore # warner/foolscap#78 def set_nurls(self, nurls: list[DecodedURL]) -> None: """Set the HTTP NURLs for this server.""" - self._nurls = [n.to_text() for n in nurls] + self._nurls = [n.to_text().encode("utf-8") for n in nurls] def remote_get_version(self): result = self._server.get_version() if self._nurls: + # Tell clients how to upgrade to HTTP protocol: result[b"http://allmydata.org/tahoe/protocols/storage/v2"] = { - ANONYMOUS_STORAGE_NURLS: self._nurls + ANONYMOUS_STORAGE_NURLS.encode("ascii"): self._nurls } return result diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 4efc845b4..b9356103b 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -93,10 +93,9 @@ from allmydata.storage.http_client import ( ReadVector, TestWriteVectors, WriteVector, TestVector, ClientException, StorageClientFactory ) +from .storage.http_common import ANONYMOUS_STORAGE_NURLS from .node import _Config -ANONYMOUS_STORAGE_NURLS = "anonymous-storage-NURLs" - # who is responsible for de-duplication? # both? @@ -333,6 +332,8 @@ class StorageFarmBroker(service.MultiService): return s handler_overrides = server.get("connections", {}) + # TODO connect and get version; if it has HTTP and we support HTTP, do + # HTTPNativeStorageServer instead. s = NativeStorageServer( server_id, server["ann"], diff --git a/src/allmydata/test/test_storage.py b/src/allmydata/test/test_storage.py index c1d6004e8..983a89666 100644 --- a/src/allmydata/test/test_storage.py +++ b/src/allmydata/test/test_storage.py @@ -33,6 +33,7 @@ from twisted.trial import unittest from twisted.internet import defer from twisted.internet.task import Clock +from hyperlink import DecodedURL from hypothesis import given, strategies, example import itertools @@ -583,6 +584,19 @@ class Server(AsyncTestCase): sv1 = ver[b'http://allmydata.org/tahoe/protocols/storage/v1'] self.assertTrue(sv1.get(b'prevents-read-past-end-of-share-data'), sv1) + def test_get_version_can_expose_nurls(self): + """ + ``FoolscapStorageServer.remote_get_version()`` includes NURLs if set. + """ + ss = FoolscapStorageServer(self.create("test_get_version_can_expose_nurls")) + v2 = b'http://allmydata.org/tahoe/protocols/storage/v2' + self.assertNotIn(v2, ss.remote_get_version()) + ss.set_nurls([DecodedURL.from_text("pb://blah"), DecodedURL.from_text("pb://baz")]) + self.assertEqual( + ss.remote_get_version()[v2], + {b"anonymous-storage-NURLs": [b"pb://blah", b"pb://baz"]} + ) + def test_declares_maximum_share_sizes(self): ss = self.create("test_declares_maximum_share_sizes") ver = ss.get_version()