From 8cf53d2d126af8b36c5496f0c02c1a47b7b5e19e Mon Sep 17 00:00:00 2001 From: Brian Warner Date: Sat, 27 Aug 2016 11:27:58 -0700 Subject: [PATCH] derive permutation seed from pubkey/server_id --- docs/configuration.rst | 50 ++++++++++++++++++----- src/allmydata/storage_client.py | 16 ++++++-- src/allmydata/test/test_storage_client.py | 35 ++++++++++++++++ 3 files changed, 87 insertions(+), 14 deletions(-) diff --git a/docs/configuration.rst b/docs/configuration.rst index 92ca7e935..7a9189797 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -687,28 +687,56 @@ The ``storage`` dictionary takes keys which are server-ids, and values which are dictionaries with two keys: ``ann`` and ``connections``. The ``ann`` value is a dictionary which will be used in lieu of the introducer announcement, so it can be populated by copying the ``ann`` dictionary from -``NODEDIR/introducer_cache.yaml``. Static servers which use the node's -default connection handlers only need a few keys: +``NODEDIR/introducer_cache.yaml``. -* the server ID, which can be any string -* a nickname, which is the string that is printed on the web interface -* the ``anonymous-storage-FURL``, which is where the server lives -* ``permutation-seed-base32``, which controls how shares are mapped to +The server-id can be any string, but ideally you should use the public key as +published by the server. Each server displays this as "Node ID:" in the +top-right corner of its "WUI" web welcome page. It can also be obtained from +other client nodes, which record it as ``key_s:`` in their +``introducer_cache.yaml`` file. The format is "v0-" followed by 52 base32 +characters like so:: + + v0-c2ng2pbrmxmlwpijn3mr72ckk5fmzk6uxf6nhowyosaubrt6y5mq + +The ``ann`` dictionary really only needs one key: + +* ``anonymous-storage-FURL``: how we connect to the server + +(note that other important keys may be added in the future, as Accounting and +HTTP-based servers are implemented) + +Optional keys include: + +* ``nickname``: the name of this server, as displayed on the Welcome page + server list +* ``permutation-seed-base32``: this controls how shares are mapped to servers. This is normally computed from the server-ID, but can be overridden to maintain the mapping for older servers which used to use - Foolscap TubIDs as server-IDs. -* more important keys may be added in the future, as Accounting and - HTTP-based servers are implemented + Foolscap TubIDs as server-IDs. If your selected server-ID cannot be parsed + as a public key, it will be hashed to compute the permutation seed. This is + fine as long as all clients use the same thing, but if they don't, then + your client will disagree with the other clients about which servers should + hold each share. This will slow downloads for everybody, and may cause + additional work or consume extra storage when repair operations don't + converge. +* anything else from the ``introducer_cache.yaml`` announcement, like + ``my-version``, which is displayed on the Welcome page server list For example, a private static server could be defined with a ``private/servers.yaml`` file like this:: storage: - my-serverid-1: + v0-4uazse3xb6uu5qpkb7tel2bm6bpea4jhuigdhqcuvvse7hugtsia: ann: nickname: my-server-1 anonymous-storage-FURL: pb://u33m4y7klhz3bypswqkozwetvabelhxt@tcp:8.8.8.8:51298/eiu2i7p6d6mm4ihmss7ieou5hac3wn6b - permutation-seed-base32: w2hqnbaa25yw4qgcvghl5psa3srpfgw3 + +Or, if you're feeling really lazy:: + + storage: + my-serverid-1: + ann: + anonymous-storage-FURL: pb://u33m4y7klhz3bypswqkozwetvabelhxt@tcp:8.8.8.8:51298/eiu2i7p6d6mm4ihmss7ieou5hac3wn6b .. _YAML: http://yaml.org/ diff --git a/src/allmydata/storage_client.py b/src/allmydata/storage_client.py index 1853a5596..784a04c9f 100644 --- a/src/allmydata/storage_client.py +++ b/src/allmydata/storage_client.py @@ -29,7 +29,7 @@ the foolscap-based server implemented in src/allmydata/storage/*.py . # 6: implement other sorts of IStorageClient classes: S3, etc -import re, time +import re, time, hashlib from zope.interface import implements from twisted.internet import defer from twisted.application import service @@ -289,8 +289,18 @@ class NativeStorageServer(service.MultiService): assert m, furl tubid_s = m.group(1).lower() self._tubid = base32.a2b(tubid_s) - assert "permutation-seed-base32" in ann, ann - ps = base32.a2b(str(ann["permutation-seed-base32"])) + if "permutation-seed-base32" in ann: + ps = base32.a2b(str(ann["permutation-seed-base32"])) + elif re.search(r'^v0-[0-9a-zA-Z]{52}$', server_id): + ps = base32.a2b(server_id[3:]) + else: + log.msg("unable to parse serverid '%(server_id)s as pubkey, " + "hashing it to get permutation-seed, " + "may not converge with other clients", + server_id=server_id, + facility="tahoe.storage_broker", + level=log.UNUSUAL, umid="qu86tw") + ps = hashlib.sha256(server_id).digest() self._permutation_seed = ps assert server_id diff --git a/src/allmydata/test/test_storage_client.py b/src/allmydata/test/test_storage_client.py index 2e7f63635..8c53ec630 100644 --- a/src/allmydata/test/test_storage_client.py +++ b/src/allmydata/test/test_storage_client.py @@ -1,3 +1,4 @@ +import hashlib from mock import Mock, patch from allmydata.util import base32 @@ -74,6 +75,40 @@ class TestStorageFarmBroker(unittest.TestCase): self.assertIdentical(s2, s) self.assertEqual(s2.get_permutation_seed(), permseed) + def test_static_permutation_seed_pubkey(self): + broker = StorageFarmBroker(True) + server_id = "v0-4uazse3xb6uu5qpkb7tel2bm6bpea4jhuigdhqcuvvse7hugtsia" + k = "4uazse3xb6uu5qpkb7tel2bm6bpea4jhuigdhqcuvvse7hugtsia" + ann = { + "anonymous-storage-FURL": "pb://abcde@nowhere/fake", + } + broker.set_static_servers({server_id: {"ann": ann}}) + s = broker.servers[server_id] + self.assertEqual(s.get_permutation_seed(), base32.a2b(k)) + + def test_static_permutation_seed_explicit(self): + broker = StorageFarmBroker(True) + server_id = "v0-4uazse3xb6uu5qpkb7tel2bm6bpea4jhuigdhqcuvvse7hugtsia" + k = "w5gl5igiexhwmftwzhai5jy2jixn7yx7" + ann = { + "anonymous-storage-FURL": "pb://abcde@nowhere/fake", + "permutation-seed-base32": k, + } + broker.set_static_servers({server_id: {"ann": ann}}) + s = broker.servers[server_id] + self.assertEqual(s.get_permutation_seed(), base32.a2b(k)) + + def test_static_permutation_seed_hashed(self): + broker = StorageFarmBroker(True) + server_id = "unparseable" + ann = { + "anonymous-storage-FURL": "pb://abcde@nowhere/fake", + } + broker.set_static_servers({server_id: {"ann": ann}}) + s = broker.servers[server_id] + self.assertEqual(s.get_permutation_seed(), + hashlib.sha256(server_id).digest()) + @inlineCallbacks def test_threshold_reached(self): introducer = Mock()