From 40d649b3b202114bc0ce46fccb194b662b4991e3 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 23 Oct 2023 09:44:11 -0400 Subject: [PATCH 01/22] Make another slowish operation non-blocking --- src/allmydata/mutable/retrieve.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/src/allmydata/mutable/retrieve.py b/src/allmydata/mutable/retrieve.py index 41c87ac59..54ada2ca9 100644 --- a/src/allmydata/mutable/retrieve.py +++ b/src/allmydata/mutable/retrieve.py @@ -928,12 +928,20 @@ class Retrieve(object): reason, ) - - def _try_to_validate_privkey(self, enc_privkey, reader, server): + @deferredutil.async_to_deferred + async def _try_to_validate_privkey(self, enc_privkey, reader, server): node_writekey = self._node.get_writekey() - alleged_privkey_s = decrypt_privkey(node_writekey, enc_privkey) - alleged_writekey = hashutil.ssk_writekey_hash(alleged_privkey_s) - if alleged_writekey != node_writekey: + + def get_privkey(): + alleged_privkey_s = decrypt_privkey(node_writekey, enc_privkey) + alleged_writekey = hashutil.ssk_writekey_hash(alleged_privkey_s) + if alleged_writekey != node_writekey: + return None + privkey, _ = rsa.create_signing_keypair_from_string(alleged_privkey_s) + return privkey + + privkey = await defer_to_thread(get_privkey) + if privkey is None: self.log("invalid privkey from %s shnum %d" % (reader, reader.shnum), level=log.WEIRD, umid="YIw4tA") @@ -950,7 +958,6 @@ class Retrieve(object): # it's good self.log("got valid privkey from shnum %d on reader %s" % (reader.shnum, reader)) - privkey, _ = rsa.create_signing_keypair_from_string(alleged_privkey_s) self._node._populate_encprivkey(enc_privkey) self._node._populate_privkey(privkey) self._need_privkey = False From 101453cd56acba66817b0c8e7f25bd6c8889c53c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Mon, 23 Oct 2023 09:57:32 -0400 Subject: [PATCH 02/22] Make operation non-blocking (assuming GIL is released) --- src/allmydata/mutable/filenode.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/allmydata/mutable/filenode.py b/src/allmydata/mutable/filenode.py index 00b31c52b..ede74d249 100644 --- a/src/allmydata/mutable/filenode.py +++ b/src/allmydata/mutable/filenode.py @@ -14,6 +14,7 @@ from allmydata.interfaces import IMutableFileNode, ICheckable, ICheckResults, \ IMutableFileVersion, IWriteable from allmydata.util import hashutil, log, consumer, deferredutil, mathutil from allmydata.util.assertutil import precondition +from allmydata.util.cputhreadpool import defer_to_thread from allmydata.uri import WriteableSSKFileURI, ReadonlySSKFileURI, \ WriteableMDMFFileURI, ReadonlyMDMFFileURI from allmydata.monitor import Monitor @@ -128,7 +129,8 @@ class MutableFileNode(object): return self - def create_with_keys(self, keypair, contents, + @deferredutil.async_to_deferred + async def create_with_keys(self, keypair, contents, version=SDMF_VERSION): """Call this to create a brand-new mutable file. It will create the shares, find homes for them, and upload the initial contents (created @@ -137,8 +139,8 @@ class MutableFileNode(object): use) when it completes. """ self._pubkey, self._privkey = keypair - self._writekey, self._encprivkey, self._fingerprint = derive_mutable_keys( - keypair, + self._writekey, self._encprivkey, self._fingerprint = await defer_to_thread( + derive_mutable_keys, keypair ) if version == MDMF_VERSION: self._uri = WriteableMDMFFileURI(self._writekey, self._fingerprint) @@ -149,7 +151,7 @@ class MutableFileNode(object): self._readkey = self._uri.readkey self._storage_index = self._uri.storage_index initial_contents = self._get_initial_contents(contents) - return self._upload(initial_contents, None) + return await self._upload(initial_contents, None) def _get_initial_contents(self, contents): if contents is None: From 0c2db2d5a87ffe2bd0693d239fad9d2b27057e2c Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 15 Nov 2023 15:53:25 -0500 Subject: [PATCH 03/22] Make sure FEC does some work --- benchmarks/conftest.py | 2 +- newsfragments/4072.feature | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 newsfragments/4072.feature diff --git a/benchmarks/conftest.py b/benchmarks/conftest.py index 926978a29..972d89b48 100644 --- a/benchmarks/conftest.py +++ b/benchmarks/conftest.py @@ -101,7 +101,7 @@ def client_node(request, grid, storage_nodes, number_of_nodes) -> Client: "client_node", needed=number_of_nodes, happy=number_of_nodes, - total=number_of_nodes, + total=number_of_nodes + 3, # Make sure FEC does some work ) ) print(f"Client node pid: {client_node.process.transport.pid}") diff --git a/newsfragments/4072.feature b/newsfragments/4072.feature new file mode 100644 index 000000000..3b0db7a02 --- /dev/null +++ b/newsfragments/4072.feature @@ -0,0 +1 @@ +Continued work to make Tahoe-LAFS take advantage of multiple CPUs. \ No newline at end of file From 4c03d931bd9f3fa0defb57dcdd5d7f41b4ae3a61 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 6 Dec 2023 15:56:58 -0500 Subject: [PATCH 04/22] Accept memoryview --- src/allmydata/crypto/aes.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/allmydata/crypto/aes.py b/src/allmydata/crypto/aes.py index ad7cfcba4..a67501eb0 100644 --- a/src/allmydata/crypto/aes.py +++ b/src/allmydata/crypto/aes.py @@ -87,8 +87,8 @@ def encrypt_data(encryptor, plaintext): """ _validate_cryptor(encryptor, encrypt=True) - if not isinstance(plaintext, six.binary_type): - raise ValueError('Plaintext must be bytes') + if not isinstance(plaintext, (six.binary_type, memoryview)): + raise ValueError(f'Plaintext must be bytes or memoryview: {type(plaintext)}') return encryptor.update(plaintext) @@ -126,8 +126,8 @@ def decrypt_data(decryptor, plaintext): """ _validate_cryptor(decryptor, encrypt=False) - if not isinstance(plaintext, six.binary_type): - raise ValueError('Plaintext must be bytes') + if not isinstance(plaintext, (six.binary_type, memoryview)): + raise ValueError(f'Plaintext must be bytes or memoryview: {type(plaintext)}') return decryptor.update(plaintext) From 9a1e73892e1b58cb6da34b205cf2438811ea9a82 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 6 Dec 2023 15:58:30 -0500 Subject: [PATCH 05/22] Run joining in a thread --- src/allmydata/mutable/retrieve.py | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/src/allmydata/mutable/retrieve.py b/src/allmydata/mutable/retrieve.py index 54ada2ca9..b4db6a092 100644 --- a/src/allmydata/mutable/retrieve.py +++ b/src/allmydata/mutable/retrieve.py @@ -4,8 +4,9 @@ Ported to Python 3. from __future__ import annotations import time - +from io import BytesIO from itertools import count + from zope.interface import implementer from twisted.internet import defer from twisted.python import failure @@ -873,11 +874,26 @@ class Retrieve(object): shares = shares[:self._required_shares] self.log("decoding segment %d" % segnum) if segnum == self._num_segments - 1: - d = defer.maybeDeferred(self._tail_decoder.decode, shares, shareids) + d = self._tail_decoder.decode(shares, shareids) else: - d = defer.maybeDeferred(self._segment_decoder.decode, shares, shareids) - def _process(buffers): - segment = b"".join(buffers) + d = self._segment_decoder.decode(shares, shareids) + + # For larger shares, this can take a few milliseconds. As such, we want + # to unblock the event loop. Even if it doesn't release the GIL, if it + # really takes too long it will implicitly release it. + def _join(buffers): + f = BytesIO() + for b in buffers: + f.write(b) + return f.getbuffer() + + @deferredutil.async_to_deferred + async def _got_buffers(buffers): + return await defer_to_thread(_join, buffers) + + d.addCallback(_got_buffers) + + def _process(segment): self.log(format="now decoding segment %(segnum)s of %(numsegs)s", segnum=segnum, numsegs=self._num_segments, From 81a5ae6f461cb615bdc2ae118b6b5223fda6b3ba Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Wed, 6 Dec 2023 16:01:14 -0500 Subject: [PATCH 06/22] Simplify --- src/allmydata/mutable/retrieve.py | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/src/allmydata/mutable/retrieve.py b/src/allmydata/mutable/retrieve.py index b4db6a092..45d7766ee 100644 --- a/src/allmydata/mutable/retrieve.py +++ b/src/allmydata/mutable/retrieve.py @@ -4,7 +4,6 @@ Ported to Python 3. from __future__ import annotations import time -from io import BytesIO from itertools import count from zope.interface import implementer @@ -879,17 +878,11 @@ class Retrieve(object): d = self._segment_decoder.decode(shares, shareids) # For larger shares, this can take a few milliseconds. As such, we want - # to unblock the event loop. Even if it doesn't release the GIL, if it - # really takes too long it will implicitly release it. - def _join(buffers): - f = BytesIO() - for b in buffers: - f.write(b) - return f.getbuffer() - + # to unblock the event loop. In newer Python b"".join() will release + # the GIL: https://github.com/python/cpython/issues/80232 @deferredutil.async_to_deferred async def _got_buffers(buffers): - return await defer_to_thread(_join, buffers) + return await defer_to_thread(lambda: b"".join(buffers)) d.addCallback(_got_buffers) From 2783bd8b7799f6c57a7100e8920bc5adf88c0a52 Mon Sep 17 00:00:00 2001 From: Itamar Turner-Trauring Date: Thu, 7 Dec 2023 16:39:30 -0500 Subject: [PATCH 07/22] Unnecessary maybeDeferred --- src/allmydata/immutable/downloader/node.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/immutable/downloader/node.py b/src/allmydata/immutable/downloader/node.py index a1ef4b485..efa3e09eb 100644 --- a/src/allmydata/immutable/downloader/node.py +++ b/src/allmydata/immutable/downloader/node.py @@ -419,7 +419,7 @@ class DownloadNode(object): def process_blocks(self, segnum, blocks): start = now() - d = defer.maybeDeferred(self._decode_blocks, segnum, blocks) + d = self._decode_blocks(segnum, blocks) d.addCallback(self._check_ciphertext_hash, segnum) def _deliver(result): log.msg(format="delivering segment(%(segnum)d)", From dd568ab6f4ebd0979320b17bb38c71b457d323f9 Mon Sep 17 00:00:00 2001 From: "Christopher R. Wood" Date: Fri, 3 May 2024 16:40:01 -0400 Subject: [PATCH 08/22] Add tests for supplying RSA private keys to mkdir --- integration/test_web.py | 122 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 122 insertions(+) diff --git a/integration/test_web.py b/integration/test_web.py index 08c6e6217..94120be92 100644 --- a/integration/test_web.py +++ b/integration/test_web.py @@ -12,11 +12,18 @@ exists anywhere, however. from __future__ import annotations import time +from base64 import urlsafe_b64encode from urllib.parse import unquote as url_unquote, quote as url_quote +from cryptography.hazmat.primitives.serialization import load_pem_private_key from twisted.internet.threads import deferToThread import allmydata.uri +from allmydata.crypto.rsa import ( + create_signing_keypair, + der_string_from_signing_key, +) +from allmydata.mutable.common import derive_mutable_keys from allmydata.util import jsonbytes as json from . import util @@ -541,3 +548,118 @@ def test_mkdir_with_children(alice): assert resp.startswith(b"URI:DIR2") cap = allmydata.uri.from_string(resp) assert isinstance(cap, allmydata.uri.DirectoryURI) + + +@run_in_thread +def test_mkdir_with_random_private_key(alice): + """ + Create a new directory with ?t=mkdir&private-key=... using a + randomly-generated RSA private key. + + The writekey and fingerprint derived from the provided RSA key + should match those of the newly-created directory capability. + """ + + privkey, pubkey = create_signing_keypair(2048) + + writekey, _, fingerprint = derive_mutable_keys((pubkey, privkey)) + + # The "private-key" parameter takes a DER-encoded RSA private key + # encoded in URL-safe base64; PEM blocks are not supported. + privkey_der = der_string_from_signing_key(privkey) + privkey_encoded = urlsafe_b64encode(privkey_der).decode("ascii") + + resp = util.web_post( + alice.process, u"uri", + params={ + u"t": "mkdir", + u"private-key": privkey_encoded, + }, + ) + assert resp.startswith(b"URI:DIR2") + + dircap = allmydata.uri.from_string(resp) + assert isinstance(dircap, allmydata.uri.DirectoryURI) + + # DirectoryURI objects lack 'writekey' and 'fingerprint' attributes + # so extract them from the enclosed WriteableSSKFileURI object. + filecap = dircap.get_filenode_cap() + assert isinstance(filecap, allmydata.uri.WriteableSSKFileURI) + + assert (writekey, fingerprint) == (filecap.writekey, filecap.fingerprint) + + +@run_in_thread +def test_mkdir_with_known_private_key(alice): + """ + Create a new directory with ?t=mkdir&private-key=... using a + known-in-advance RSA private key. + + The writekey and fingerprint derived from the provided RSA key + should match those of the newly-created directory capability. + In addition, because the writekey and fingerprint are derived + deterministically, given the same RSA private key, the resultant + directory capability should always be the same. + """ + # Randomly generated with `openssl genrsa -out privkey.pem 2048` + privkey_pem = """-----BEGIN RSA PRIVATE KEY----- +MIIEowIBAAKCAQEAoa9i8v9YIzb+3yRHyXLm4j1eWK9lQc6lFwoQhik8y+joD+5A +v73OlDZAcn6vzlU72vwrJ1f4o54nEVm0rhNrhwCsiHCdxxEDEoqZ8w/19vc4hWj4 +SYwGirhcnyb2ysZSV8v9Lm5HiFe5zZM4jzCzf2rzt0YRlZZj9nhSglaiHZ9BE2e0 +vzOl6GePDz6yS4jbh2RsPsDQtqXNOqZwfGUd+iTsbSxXcm8+rNrT1VAbx6+1Sr0r +aDyc/jp8S1JwJ0ofJLsU3Pb6DYazFf12CNTsrKF1L0hAsbN8v2DSunZIQqQLQGfp +0hnNO9V8q9FjvVu8XY/HhgoTvtESU3vuq+BnIwIDAQABAoIBAGpWDP+/y9mtK8bZ +95SXyx10Ov6crD2xiIY0ilWR/XgmP6lqio8QaDK104D5rOpIyErnmgIQK2iAdTVG +CDyMbSWm3dIGLt5jY9/n5AQltSCtyzCCrvi/7PWC9vd9Csal1DYF5QeKY+VZvMtl +Tcduwj7EunEI1jvJYwkQbUNncsuDi+88/JNwa8DJp1IrR4goxNflGl7mNzfq49re +lhSyezfLSTZKDa3A6sYnNFAAOy82iXZuLXCqKuwRuaiFFilB0R0/egzBSUeBwMJk +sS+SvHHXwv9HsYt4pYiiZFm8HxB4NKYtdpHpvJVJcG9vOXjewnA5YHWVDJsrBfu6 +0kPgbcECgYEA0bqfX2Vc6DizwjWVn9yVlckjQNGTnwf/B9eGW2MgTn6YADe0yjFm +KCtr34hEZc/hv3kBnoLOqSvZJiser8ve3SmwxfmpjEfJdIgA5J5DbCEGBiDm9PMy +0lYsfjykzYykehdasb8f4xd+SPMuTC/CFb1MCTlohex7qn7Xt9IskBECgYEAxVtF +iXwFJPQUil2bSFGnxtaI/8ijypLOkP3CyuVnEcbMt74jDt1hdooRxjQ9VVlg7r7i +EvebPKMukWxdVcQ/38i97oB/oN7MIH0QBCDWTdTQokuNQSEknGLouj6YtLAWRcyJ +9DDENSaGtP42le5dD60hZc732jN09fGxNa6gN/MCgYB5ux98CGJ3q0mzBNUW17q/ +GOLsYXiUitidHZyveIas6M+i+LJn1WpdEG7pbLd+fL2kHEEzVutKx9efTtHd6bAu +oF8pWfLuKFCm4bXa/H1XyocrkXdcX7h0222xy9NAN0zUTK/okW2Zqu4yu2t47xNw ++NGkXPztFsjkugDNgiE5cQKBgQDDy/BqHPORnOIAACw9jF1SpKcYdPsiz5FGQawO +1ZbzCPMzW9y2M6YtD3/gzxUGZv0G/7OUs7h8aTybJBJZM7FXGHZud2ent0J2/Px1 +zAow/3DZgvEp63LCAFL5635ezM/cAbff3r3aKVW9nPOUvf3vvokC01oMTb68/kMc +ihoERwKBgFsoRUrgGPSfG1UZt8BpIXbG/8qfoy/Vy77BRqvJ6ZpdM9RPqdAl7Sih +cdqfxs8w0NVvj+gvM/1CGO0J9lZW2f1J81haIoyUpiITFdoyzLKXLhMSbaF4Y7Hn +yC/N5w3cCLa2LLKoLG8hagFDlXBGSmpT1zgKBk4YxNn6CLdMSzPR +-----END RSA PRIVATE KEY----- +""" + + privkey = load_pem_private_key( + privkey_pem.encode("ascii"), password=None + ) + pubkey = privkey.public_key() + + writekey, _, fingerprint = derive_mutable_keys((pubkey, privkey)) + + # The "private-key" parameter takes a DER-encoded RSA private key + # encoded in URL-safe base64; PEM blocks are not supported. + privkey_der = der_string_from_signing_key(privkey) + privkey_encoded = urlsafe_b64encode(privkey_der).decode("ascii") + + resp = util.web_post( + alice.process, u"uri", + params={ + u"t": "mkdir", + u"private-key": privkey_encoded, + }, + ) + assert resp.startswith(b"URI:DIR2") + + dircap = allmydata.uri.from_string(resp) + assert isinstance(dircap, allmydata.uri.DirectoryURI) + + # DirectoryURI objects lack 'writekey' and 'fingerprint' attributes + # so extract them from the enclosed WriteableSSKFileURI object. + filecap = dircap.get_filenode_cap() + assert isinstance(filecap, allmydata.uri.WriteableSSKFileURI) + + assert (writekey, fingerprint) == (filecap.writekey, filecap.fingerprint) + + assert resp == b"URI:DIR2:3oo7j7f7qqxnet2z2lf57ucup4:cpktmsxlqnd5yeekytxjxvff5e6d6fv7py6rftugcndvss7tzd2a" From 9c2362853db0cbb9dee8529977f70a6c9713d5f3 Mon Sep 17 00:00:00 2001 From: "Christopher R. Wood" Date: Fri, 3 May 2024 16:55:38 -0400 Subject: [PATCH 09/22] Allow supplying keypair when creating mutable dirs --- src/allmydata/client.py | 14 ++++++++++++-- src/allmydata/nodemaker.py | 11 +++++++++-- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index 03bf609e9..6edbf7eeb 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -1125,8 +1125,18 @@ class _Client(node.Node, pollmixin.PollMixin): # may get an opaque node if there were any problems. return self.nodemaker.create_from_cap(write_uri, read_uri, deep_immutable=deep_immutable, name=name) - def create_dirnode(self, initial_children=None, version=None): - d = self.nodemaker.create_new_mutable_directory(initial_children, version=version) + def create_dirnode( + self, + initial_children=None, + version=None, + *, + unique_keypair: tuple[rsa.PublicKey, rsa.PrivateKey] | None = None + ): + d = self.nodemaker.create_new_mutable_directory( + initial_children, + version=version, + keypair=unique_keypair, + ) return d def create_immutable_dirnode(self, children, convergence=None): diff --git a/src/allmydata/nodemaker.py b/src/allmydata/nodemaker.py index 39663bda9..6e8700cff 100644 --- a/src/allmydata/nodemaker.py +++ b/src/allmydata/nodemaker.py @@ -135,7 +135,13 @@ class NodeMaker(object): d.addCallback(lambda res: n) return d - def create_new_mutable_directory(self, initial_children=None, version=None): + def create_new_mutable_directory( + self, + initial_children=None, + version=None, + *, + keypair: tuple[PublicKey, PrivateKey] | None = None, + ): if initial_children is None: initial_children = {} for (name, (node, metadata)) in initial_children.items(): @@ -145,7 +151,8 @@ class NodeMaker(object): d = self.create_mutable_file(lambda n: MutableData(pack_children(initial_children, n.get_writekey())), - version=version) + version=version, + keypair=keypair) d.addCallback(self._create_dirnode) return d From b93a39fdc2b6353eeb5c5f704159731fd318a371 Mon Sep 17 00:00:00 2001 From: "Christopher R. Wood" Date: Sat, 4 May 2024 09:34:43 -0400 Subject: [PATCH 10/22] Allow POST /uri?t=mkdir to accept `private-key` --- src/allmydata/web/unlinked.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/web/unlinked.py b/src/allmydata/web/unlinked.py index 2c7be6f30..a44e7fcb5 100644 --- a/src/allmydata/web/unlinked.py +++ b/src/allmydata/web/unlinked.py @@ -160,7 +160,7 @@ def POSTUnlinkedCreateDirectory(req, client): mt = None if file_format: mt = get_mutable_type(file_format) - d = client.create_dirnode(version=mt) + d = client.create_dirnode(version=mt, unique_keypair=get_keypair(req)) redirect = get_arg(req, "redirect_to_result", "false") if boolean_of_arg(redirect): def _then_redir(res): From 6c7ffbe30d458b0bf2029b922bacb27555b09bbe Mon Sep 17 00:00:00 2001 From: "Christopher R. Wood" Date: Tue, 7 May 2024 14:37:42 -0400 Subject: [PATCH 11/22] Allow mkdir-with-children to accept `private-key` --- src/allmydata/web/unlinked.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/allmydata/web/unlinked.py b/src/allmydata/web/unlinked.py index a44e7fcb5..26c41c7be 100644 --- a/src/allmydata/web/unlinked.py +++ b/src/allmydata/web/unlinked.py @@ -178,7 +178,7 @@ def POSTUnlinkedCreateDirectoryWithChildren(req, client): req.content.seek(0) kids_json = req.content.read() kids = convert_children_json(client.nodemaker, kids_json) - d = client.create_dirnode(initial_children=kids) + d = client.create_dirnode(initial_children=kids, unique_keypair=get_keypair(req)) redirect = get_arg(req, "redirect_to_result", "false") if boolean_of_arg(redirect): def _then_redir(res): From 31b8f195db99b9b344b2bcbc9ab8ddf937ca4c6c Mon Sep 17 00:00:00 2001 From: "Christopher R. Wood" Date: Tue, 7 May 2024 14:49:40 -0400 Subject: [PATCH 12/22] Add test for mkdir-with-children with `private-key` --- integration/test_web.py | 93 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 93 insertions(+) diff --git a/integration/test_web.py b/integration/test_web.py index 94120be92..7e715122d 100644 --- a/integration/test_web.py +++ b/integration/test_web.py @@ -663,3 +663,96 @@ yC/N5w3cCLa2LLKoLG8hagFDlXBGSmpT1zgKBk4YxNn6CLdMSzPR assert (writekey, fingerprint) == (filecap.writekey, filecap.fingerprint) assert resp == b"URI:DIR2:3oo7j7f7qqxnet2z2lf57ucup4:cpktmsxlqnd5yeekytxjxvff5e6d6fv7py6rftugcndvss7tzd2a" + + +@run_in_thread +def test_mkdir_with_children_and_random_private_key(alice): + """ + Create a new directory with ?t=mkdir-with-children&private-key=... + using a randomly-generated RSA private key. + + The writekey and fingerprint derived from the provided RSA key + should match those of the newly-created directory capability. + """ + + # create a file to put in our directory + FILE_CONTENTS = u"some file contents\n" * 500 + resp = requests.put( + util.node_url(alice.process.node_dir, u"uri"), + data=FILE_CONTENTS, + ) + filecap = resp.content.strip() + + # create a (sub) directory to put in our directory + resp = requests.post( + util.node_url(alice.process.node_dir, u"uri"), + params={ + u"t": u"mkdir", + } + ) + # (we need both the read-write and read-only URIs I guess) + dircap = resp.content + dircap_obj = allmydata.uri.from_string(dircap) + dircap_ro = dircap_obj.get_readonly().to_string() + + # create json information about our directory + meta = { + "a_file": [ + "filenode", { + "ro_uri": filecap, + "metadata": { + "ctime": 1202777696.7564139, + "mtime": 1202777696.7564139, + "tahoe": { + "linkcrtime": 1202777696.7564139, + "linkmotime": 1202777696.7564139 + } + } + } + ], + "some_subdir": [ + "dirnode", { + "rw_uri": dircap, + "ro_uri": dircap_ro, + "metadata": { + "ctime": 1202778102.7589991, + "mtime": 1202778111.2160511, + "tahoe": { + "linkcrtime": 1202777696.7564139, + "linkmotime": 1202777696.7564139 + } + } + } + ] + } + + privkey, pubkey = create_signing_keypair(2048) + + writekey, _, fingerprint = derive_mutable_keys((pubkey, privkey)) + + # The "private-key" parameter takes a DER-encoded RSA private key + # encoded in URL-safe base64; PEM blocks are not supported. + privkey_der = der_string_from_signing_key(privkey) + privkey_encoded = urlsafe_b64encode(privkey_der).decode("ascii") + + # create a new directory with one file and one sub-dir (all-at-once) + # with the supplied RSA private key + resp = util.web_post( + alice.process, u"uri", + params={ + u"t": "mkdir-with-children", + u"private-key": privkey_encoded, + }, + data=json.dumps(meta), + ) + assert resp.startswith(b"URI:DIR2") + + dircap = allmydata.uri.from_string(resp) + assert isinstance(dircap, allmydata.uri.DirectoryURI) + + # DirectoryURI objects lack 'writekey' and 'fingerprint' attributes + # so extract them from the enclosed WriteableSSKFileURI object. + filecap = dircap.get_filenode_cap() + assert isinstance(filecap, allmydata.uri.WriteableSSKFileURI) + + assert (writekey, fingerprint) == (filecap.writekey, filecap.fingerprint) From 2ef6da5c4e1c04d50869e140447775ec94893ac2 Mon Sep 17 00:00:00 2001 From: "Christopher R. Wood" Date: Tue, 7 May 2024 15:15:01 -0400 Subject: [PATCH 13/22] Add test for mkdir-with-children with known `private-key` --- integration/test_web.py | 132 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 132 insertions(+) diff --git a/integration/test_web.py b/integration/test_web.py index 7e715122d..06fc36f1c 100644 --- a/integration/test_web.py +++ b/integration/test_web.py @@ -756,3 +756,135 @@ def test_mkdir_with_children_and_random_private_key(alice): assert isinstance(filecap, allmydata.uri.WriteableSSKFileURI) assert (writekey, fingerprint) == (filecap.writekey, filecap.fingerprint) + + +@run_in_thread +def test_mkdir_with_children_and_known_private_key(alice): + """ + Create a new directory with ?t=mkdir-with-children&private-key=... + using a known-in-advance RSA private key. + + + The writekey and fingerprint derived from the provided RSA key + should match those of the newly-created directory capability. + In addition, because the writekey and fingerprint are derived + deterministically, given the same RSA private key, the resultant + directory capability should always be the same. + """ + + # create a file to put in our directory + FILE_CONTENTS = u"some file contents\n" * 500 + resp = requests.put( + util.node_url(alice.process.node_dir, u"uri"), + data=FILE_CONTENTS, + ) + filecap = resp.content.strip() + + # create a (sub) directory to put in our directory + resp = requests.post( + util.node_url(alice.process.node_dir, u"uri"), + params={ + u"t": u"mkdir", + } + ) + # (we need both the read-write and read-only URIs I guess) + dircap = resp.content + dircap_obj = allmydata.uri.from_string(dircap) + dircap_ro = dircap_obj.get_readonly().to_string() + + # create json information about our directory + meta = { + "a_file": [ + "filenode", { + "ro_uri": filecap, + "metadata": { + "ctime": 1202777696.7564139, + "mtime": 1202777696.7564139, + "tahoe": { + "linkcrtime": 1202777696.7564139, + "linkmotime": 1202777696.7564139 + } + } + } + ], + "some_subdir": [ + "dirnode", { + "rw_uri": dircap, + "ro_uri": dircap_ro, + "metadata": { + "ctime": 1202778102.7589991, + "mtime": 1202778111.2160511, + "tahoe": { + "linkcrtime": 1202777696.7564139, + "linkmotime": 1202777696.7564139 + } + } + } + ] + } + + # Randomly generated with `openssl genrsa -out privkey.pem 2048` + privkey_pem = """-----BEGIN RSA PRIVATE KEY----- +MIIEowIBAAKCAQEA2PL5Ry2BGuuUtRJa20WS0fwBOqVIVSXDVuSvZFYTT1Xji19J +q+ohHcFnIIYHAq0zQG+NgNjK5rogY/5TfbwIhfwLufleeAdL9jXTfxan0o/wwFA1 +DAIHcYsTEYI2dfQe4acOLFY6/Hh6iXCbHvSzzUnEmYkgwCAZvc0v/lD8pMnz/6gQ +2nJnAASfFovcAvfr1T+MZzLJGQem3f2IFp1frurQyFmzFRtZMO5B9PDSsFG4yJVf +cz0iSP8wlc9QydImmJGRvu4xEOkx/55B/XaUdb6CIGpCTkLsDOlImvZt9UHDSgXq +qcE/T7SYMIXqbep64tJw9enjomH+n1KVh9UA2wIDAQABAoIBABCSTrQ/J5N010EV +i9cf810S0M03/tRyM/+ZLESPxp3Sw7TLrIbzNWBee5AibLqpnDaZzsc+yBDjusGo +lZwPFt+VJxgnki288PJ3nhYhFuSglhU6izLFnOfxZZ16wsozwYAfEJgWZh8O3N1O +uqqcqndN4TSRIu1KBm1XFQlqCkJT/stzYjO4k1vhgZT4pqhYRdx7q7FAap4v+sNs +Svhm1blvOXlyeumAbFBdGFttpTxIOGRzI1bp00jcLK4rgssTTxNyEiVu4oJhQY/k +0CptSUzpGio8DZ0/8bNnKCkw8YATUWJZQgSmKraRwAYMMR/SZa7WqjEc2KRTj6xQ +pHmYwZECgYEA700a/7ur8+EwTSulLgDveAOtTV0xEbhuq6cJQgNrEp2rbFqie6FX +g/YJKzEpEnUvj/yOzhEcw3CdQDUaxndlqY87QIhUWMcsnfMPsM1FjhmfksR8s3TF +WZNqa0RAKmcRoLohGclSvRV2OVU8+10mLUwJfR86Nl5+auR3LxWLyB8CgYEA6BaR +r+Z7oTlgkdEDVhnQ58Msktv58y28N+VIbYS79bV01jqUUlogm5uTvdvq5nyENXHx +gnK88mVzWYBMk83D01HlOC5DhpspTVEQQG2V/If6KZa56mxiHP3Mab9jLew9w/kA +g6l/04ATSA8g4i2H/Bz0eEyPEBt6o/+SO0Xv38UCgYEAyTTLvrrNmgF922UXPdcL +gp2U2bfBymSIqUuJPTgij0SDHlgWxlyieRImI2ryXdKqayav7BP3W10U2yfLm5RI +pokICPqX8Q2HNkdoqf/uu8xPn9gWAc3tIaQRlp+MVBrVd48IxeXA67tf7FT/MVrg +/rUwRUQ8bfqF0NrIW46COYECgYAYDJamGoT/DNoD4hutZVlvWpsY0LCS0U9qn1ik ++Jcde+MSe9l4uxwb48AocUxi+84bV6ZF9Su9FmQghxnoSu8ay6ar7qdSoGtkNp0v +f+uF0nVKr/Kt5vM3u9jdsFZPoOY5k2jJO9wiB2h4FBE9PqiTqFBw0sYUTjSkH8yA +VdvoXQKBgFqCC8Y82eVf0/ORGTgG/KhZ72WFQKHyAeryvoLuadZ6JAI6qW9U1l9P +18SMnCO+opGN5GH2Qx7gdg17KzWzTW1gnbv0QUPNnnYEJU8VYMelNuKa8tmNgFH7 +inAwsxbbWoR08ai4exzbJrNrLpDRg5ih2wMtknN6D8m+EAvBC/Gj +-----END RSA PRIVATE KEY----- +""" + + privkey = load_pem_private_key( + privkey_pem.encode("ascii"), password=None + ) + pubkey = privkey.public_key() + + writekey, _, fingerprint = derive_mutable_keys((pubkey, privkey)) + + # The "private-key" parameter takes a DER-encoded RSA private key + # encoded in URL-safe base64; PEM blocks are not supported. + privkey_der = der_string_from_signing_key(privkey) + privkey_encoded = urlsafe_b64encode(privkey_der).decode("ascii") + + # create a new directory with one file and one sub-dir (all-at-once) + # with the supplied RSA private key + resp = util.web_post( + alice.process, u"uri", + params={ + u"t": "mkdir-with-children", + u"private-key": privkey_encoded, + }, + data=json.dumps(meta), + ) + assert resp.startswith(b"URI:DIR2") + + dircap = allmydata.uri.from_string(resp) + assert isinstance(dircap, allmydata.uri.DirectoryURI) + + # DirectoryURI objects lack 'writekey' and 'fingerprint' attributes + # so extract them from the enclosed WriteableSSKFileURI object. + filecap = dircap.get_filenode_cap() + assert isinstance(filecap, allmydata.uri.WriteableSSKFileURI) + + assert (writekey, fingerprint) == (filecap.writekey, filecap.fingerprint) + + assert resp == b"URI:DIR2:ppwzpwrd37xi7tpribxyaa25uy:imdws47wwpzfkc5vfllo4ugspb36iit4cqps6ttuhaouc66jb2da" From 9d10bda2a04e472adb16082e677ef1c3f231a130 Mon Sep 17 00:00:00 2001 From: "Christopher R. Wood" Date: Mon, 13 May 2024 21:37:00 -0400 Subject: [PATCH 14/22] Document "private-key=" argument for mkdir --- docs/frontends/webapi.rst | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/docs/frontends/webapi.rst b/docs/frontends/webapi.rst index 77ce11974..b581d7aeb 100644 --- a/docs/frontends/webapi.rst +++ b/docs/frontends/webapi.rst @@ -446,6 +446,16 @@ Creating a New Directory given, the directory's format is determined by the default mutable file format, as configured on the Tahoe-LAFS node responding to the request. + In addition, an optional "private-key=" argument is supported which, if given, + specifies the underlying signing key to be used when creating the directory. + This value must be a DER-encoded 2048-bit RSA private key in urlsafe base64 + encoding. Because this key can be used to derive the write capability for the + associated directory, additional care should be taken to ensure that the key is + unique, that it is kept confidential, and that it was derived from an + appropriate (high-entropy) source of randomness. If this argument is omitted + (the default behavior), Tahoe-LAFS will generate an appropriate signing key + using the underlying operating system's source of entropy. + ``POST /uri?t=mkdir-with-children`` Create a new directory, populated with a set of child nodes, and return its @@ -453,7 +463,8 @@ Creating a New Directory any other directory: the returned write-cap is the only reference to it. The format of the directory can be controlled with the format= argument in - the query string, as described above. + the query string and a signing key can be specified with the private-key= + argument, as described above. Initial children are provided as the body of the POST form (this is more efficient than doing separate mkdir and set_children operations). If the From 5a485545f631334e60f6d6c1c747f0469f70f7d8 Mon Sep 17 00:00:00 2001 From: "Christopher R. Wood" Date: Mon, 13 May 2024 21:47:48 -0400 Subject: [PATCH 15/22] Add news fragment --- newsfragments/4094.feature | 1 + 1 file changed, 1 insertion(+) create mode 100644 newsfragments/4094.feature diff --git a/newsfragments/4094.feature b/newsfragments/4094.feature new file mode 100644 index 000000000..85c98f3d5 --- /dev/null +++ b/newsfragments/4094.feature @@ -0,0 +1 @@ +Mutable directories can now be created with a pre-determined "signature key" via the web API using the "private-key=..." parameter. The "private-key" value must be a DER-encoded 2048-bit RSA private key in urlsafe base64 encoding. From 23af93cff74540cc9a487bf7190b54a2ff40155e Mon Sep 17 00:00:00 2001 From: "Christopher R. Wood" Date: Mon, 13 May 2024 22:30:04 -0400 Subject: [PATCH 16/22] Assert/test types of `privkey` and `pubkey` vars And appease type-checkers. --- integration/test_web.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/integration/test_web.py b/integration/test_web.py index 06fc36f1c..9f0c20c6b 100644 --- a/integration/test_web.py +++ b/integration/test_web.py @@ -22,6 +22,8 @@ import allmydata.uri from allmydata.crypto.rsa import ( create_signing_keypair, der_string_from_signing_key, + PrivateKey, + PublicKey, ) from allmydata.mutable.common import derive_mutable_keys from allmydata.util import jsonbytes as json @@ -634,7 +636,9 @@ yC/N5w3cCLa2LLKoLG8hagFDlXBGSmpT1zgKBk4YxNn6CLdMSzPR privkey = load_pem_private_key( privkey_pem.encode("ascii"), password=None ) + assert isinstance(privkey, PrivateKey) pubkey = privkey.public_key() + assert isinstance(pubkey, PublicKey) writekey, _, fingerprint = derive_mutable_keys((pubkey, privkey)) @@ -856,7 +860,9 @@ inAwsxbbWoR08ai4exzbJrNrLpDRg5ih2wMtknN6D8m+EAvBC/Gj privkey = load_pem_private_key( privkey_pem.encode("ascii"), password=None ) + assert isinstance(privkey, PrivateKey) pubkey = privkey.public_key() + assert isinstance(pubkey, PublicKey) writekey, _, fingerprint = derive_mutable_keys((pubkey, privkey)) From ed2e93582c591b275ad1a9b91f260da9a3c77b66 Mon Sep 17 00:00:00 2001 From: "Christopher R. Wood" Date: Fri, 17 May 2024 10:29:32 -0400 Subject: [PATCH 17/22] Add tests for creating dirnodes with given keypair --- src/allmydata/test/test_dirnode.py | 113 ++++++++++++++++++++++++++++- 1 file changed, 112 insertions(+), 1 deletion(-) diff --git a/src/allmydata/test/test_dirnode.py b/src/allmydata/test/test_dirnode.py index 30fba005f..8cfc02a9a 100644 --- a/src/allmydata/test/test_dirnode.py +++ b/src/allmydata/test/test_dirnode.py @@ -11,6 +11,7 @@ from twisted.internet import defer from twisted.internet.interfaces import IConsumer from allmydata import uri, dirnode from allmydata.client import _Client +from allmydata.crypto.rsa import create_signing_keypair from allmydata.immutable import upload from allmydata.immutable.literal import LiteralFileNode from allmydata.interfaces import IImmutableFileNode, IMutableFileNode, \ @@ -19,16 +20,25 @@ from allmydata.interfaces import IImmutableFileNode, IMutableFileNode, \ IDeepCheckResults, IDeepCheckAndRepairResults, \ MDMF_VERSION, SDMF_VERSION from allmydata.mutable.filenode import MutableFileNode -from allmydata.mutable.common import UncoordinatedWriteError +from allmydata.mutable.common import ( + UncoordinatedWriteError, + derive_mutable_keys, +) from allmydata.util import hashutil, base32 from allmydata.util.netstring import split_netstring from allmydata.monitor import Monitor from allmydata.test.common import make_chk_file_uri, make_mutable_file_uri, \ ErrorMixin +from allmydata.test.mutable.util import ( + FakeStorage, + make_nodemaker_with_peers, + make_peer, +) from allmydata.test.no_network import GridTestMixin from allmydata.unknown import UnknownNode, strip_prefix_for_ro from allmydata.nodemaker import NodeMaker from base64 import b32decode +from cryptography.hazmat.primitives.serialization import load_pem_private_key import allmydata.test.common_util as testutil from hypothesis import given @@ -1978,3 +1988,104 @@ class Adder(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin): d.addCallback(_test_adder) return d + + +class DeterministicDirnode(testutil.ReallyEqualMixin, testutil.ShouldFailMixin, unittest.TestCase): + def setUp(self): + # Copied from allmydata.test.mutable.test_filenode + super(DeterministicDirnode, self).setUp() + self._storage = FakeStorage() + self._peers = list( + make_peer(self._storage, n) + for n + in range(10) + ) + self.nodemaker = make_nodemaker_with_peers(self._peers) + + async def test_create_with_random_keypair(self): + """ + Create a dirnode using a random RSA keypair. + + The writekey and fingerprint of the enclosed mutable filecap + should match those derived from the given keypair. + """ + privkey, pubkey = create_signing_keypair(2048) + writekey, _, fingerprint = derive_mutable_keys((pubkey, privkey)) + + node = await self.nodemaker.create_new_mutable_directory( + keypair=(pubkey, privkey) + ) + self.failUnless(isinstance(node, dirnode.DirectoryNode)) + + dircap = uri.from_string(node.get_uri()) + self.failUnless(isinstance(dircap, uri.DirectoryURI)) + + filecap = dircap.get_filenode_cap() + self.failUnless(isinstance(filecap, uri.WriteableSSKFileURI)) + + self.failUnlessReallyEqual(filecap.writekey, writekey) + self.failUnlessReallyEqual(filecap.fingerprint, fingerprint) + + async def test_create_with_known_keypair(self): + """ + Create a dirnode using a known RSA keypair. + + The writekey and fingerprint of the enclosed mutable filecap + should match those derived from the given keypair. Because + these values are derived deterministically, given the same + keypair, the resulting filecap should also always be the same. + """ + # Randomly generated with `openssl genrsa -out privkey.pem 2048` + privkey_pem = """-----BEGIN RSA PRIVATE KEY----- +MIIEowIBAAKCAQEAygMjLBKayDEioOZap2syJhUlqI7Dkk4zV5TfVxlQFO7bR410 +eJRJY1rHGIeZxQPjytsSJvqlYEJrvvVNdhi6XN/6NA3RFL6pDTHkYyM3qbrXqlYC +HUlkS2JAZzIFRizl6nG11yIbHjPsoG+vGSjGSzVIiOP4NeIssYLpoASTIppdZxy+ +syZ6zSmPhZu7W9X73aupLjFrIZpjeKfO2+GfUwEzAH0HckLIgJpQ+vK3sqbSik/2 +1oZK33M8uvtdmba7D3uJXmxWMTJ7oyFLDpDOMl7HSUv1lZY2O2qiDPYfGDUM1BRp +6blxE+BA2INr9NO4A4H8pzhikFnaFnkpH/AxowIDAQABAoIBABprXJ8386w42NmI +JtT8bPuUCm/H9AXfWlGa87aVZebG8kCiXFgktJBc3+ryWQbuIk12ZyJX52b2aNb5 +h97pDv50gGlsYSrAYKWMH91jTrVQ7UGmq/IelhJR0DBu10e9OXh21JxFJpzFl63H +zXOR5JUTa+ATSHPrl4LDp0A5OPDuWbBWa64yx7gUI9/tljbndplCrPjmIE6+h10M +sqxW5oJpLnZpWc73QQUTuPIr+A7fLgGJYHnyCFUu9OW4ZnxNEI3/wNHPvoxkYuHN +2qVonFESiAx9mBv7JzQ7X2KIB8doY3KL6S7sAKi/i/aP7EDJ9QEtl3BR3M8/XP8E +KJVORWECgYEA8Vbw75+aVMxHUl9BJc1zESxqVvr+R0NBqMO47CBj39sTJkXY37O3 +A7j4dzCorI0NaB7Jr+AI2ZZu9CaR31Y2mhAGbNLBPK8yn0Z7iWyDIqOW1OpMDs35 +h2CI1pFLjx1a3PzhsQdzZ68izWKYBdTs2scaFz/ntaPwwPEwORaMDZECgYEA1kie +YfMRJ2GwzvbR35WvEMhVxhnmA6yuRL15Pkb1WDR3iWGM0ld/u3N4sRVCx1nU4wk/ +MMqCRdm4JaxqzR/hl8+/sp3Aai15ecqR+F+ecwbbB2XKVHfi1nqClivYnB+GgCh1 +bQYUd9LT80sIQdBEW5MBdbMFnOkt+1sSpjf1wfMCgYBAavlyrIJQQhqDdSN5iKY/ +HkDgKKy4rs4W0u9IL7kY5mvtGlWyGFEwcC35+oX7UMcUVKt3A3C5S3sgNi9XkraO +VtqwL20e2pDDjNeqrcku9MVs3YEhrn79UJoV08B8WdSICgPf8eIu+cNrWPbFD7mN +B/oB3K/nfvPjPD2n70nA0QKBgGWJN3NWR9SPV8ZZ8gyt0qxzISGjd/hZxKHR3jeC +TBMlmVbBoIay61WZW6EdX+0yRcvmv8iQzLXoendvgZP8/VqAGGe8lEY7kgoB0LUO +Kfh7USHqO7tWq2fR2TrrP9KKpaLoiOvGK8CzZ7cq4Ji+5QU3XUO2NnypiR5Hg0i7 +z3m9AoGBAIEXtoSR9OTwdmrdIQn3vsaFOkN5pyYfvAvdeZ+7wwMg/ZOwhStwctbI +Um7XqocXU+8f/gjczgLgMJj+zqr+QDH5n4vSTUMPeN0gIugI9UwWnc2rhbRCgDdY +W6SwPQGDuGoUa5PxjggkyevUUmtXvGG9jnkt9kozQOA0lOF1vbw/ +-----END RSA PRIVATE KEY----- +""" + privkey = load_pem_private_key( + privkey_pem.encode("ascii"), password=None + ) + pubkey = privkey.public_key() + writekey, _, fingerprint = derive_mutable_keys((pubkey, privkey)) + + node = await self.nodemaker.create_new_mutable_directory( + keypair=(pubkey, privkey) + ) + self.failUnless(isinstance(node, dirnode.DirectoryNode)) + + dircap = uri.from_string(node.get_uri()) + self.failUnless(isinstance(dircap, uri.DirectoryURI)) + + filecap = dircap.get_filenode_cap() + self.failUnless(isinstance(filecap, uri.WriteableSSKFileURI)) + + self.failUnlessReallyEqual(filecap.writekey, writekey) + self.failUnlessReallyEqual(filecap.fingerprint, fingerprint) + + self.failUnlessReallyEqual( + # Despite being named "to_string", this actually returns bytes.. + dircap.to_string(), + b'URI:DIR2:n4opqgewgcn4mddu4oiippaxru:ukpe4z6xdlujdpguoabergyih3bj7iaafukdqzwthy2ytdd5bs2a' + ) From 692be000a8b467fd50bb38727e92d07db249d481 Mon Sep 17 00:00:00 2001 From: "Christopher R. Wood" Date: Thu, 30 May 2024 15:44:21 -0400 Subject: [PATCH 18/22] Document converting key to DER-encoded urlsafe b64 --- docs/frontends/webapi.rst | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/docs/frontends/webapi.rst b/docs/frontends/webapi.rst index b581d7aeb..baffa412d 100644 --- a/docs/frontends/webapi.rst +++ b/docs/frontends/webapi.rst @@ -449,7 +449,12 @@ Creating a New Directory In addition, an optional "private-key=" argument is supported which, if given, specifies the underlying signing key to be used when creating the directory. This value must be a DER-encoded 2048-bit RSA private key in urlsafe base64 - encoding. Because this key can be used to derive the write capability for the + encoding. (To convert an existing PEM-encoded RSA key file into the format + required, the following commands may be used -- assuming a modern UNIX-like + environment with common tools already installed: + ``openssl rsa -in key.pem -outform der | base64 -w 0 -i - | tr '+/' '-_'``) + + Because this key can be used to derive the write capability for the associated directory, additional care should be taken to ensure that the key is unique, that it is kept confidential, and that it was derived from an appropriate (high-entropy) source of randomness. If this argument is omitted From a30a7cb4e63415e0736e0a243215843cd58248f8 Mon Sep 17 00:00:00 2001 From: "Christopher R. Wood" Date: Thu, 30 May 2024 15:48:43 -0400 Subject: [PATCH 19/22] Factor out inline test keys into "data" directory --- integration/test_web.py | 77 +++---------------- .../test/data/openssl-rsa-2048-2.txt | 27 +++++++ .../test/data/openssl-rsa-2048-3.txt | 27 +++++++ .../test/data/openssl-rsa-2048-4.txt | 27 +++++++ src/allmydata/test/test_dirnode.py | 36 +-------- 5 files changed, 96 insertions(+), 98 deletions(-) create mode 100644 src/allmydata/test/data/openssl-rsa-2048-2.txt create mode 100644 src/allmydata/test/data/openssl-rsa-2048-3.txt create mode 100644 src/allmydata/test/data/openssl-rsa-2048-4.txt diff --git a/integration/test_web.py b/integration/test_web.py index 9f0c20c6b..6ea365017 100644 --- a/integration/test_web.py +++ b/integration/test_web.py @@ -17,6 +17,7 @@ from urllib.parse import unquote as url_unquote, quote as url_quote from cryptography.hazmat.primitives.serialization import load_pem_private_key from twisted.internet.threads import deferToThread +from twisted.python.filepath import FilePath import allmydata.uri from allmydata.crypto.rsa import ( @@ -37,6 +38,10 @@ from bs4 import BeautifulSoup import pytest_twisted + +DATA_PATH = FilePath(__file__).parent().sibling("src").child("allmydata").child("test").child("data") + + @run_in_thread def test_index(alice): """ @@ -603,39 +608,9 @@ def test_mkdir_with_known_private_key(alice): deterministically, given the same RSA private key, the resultant directory capability should always be the same. """ - # Randomly generated with `openssl genrsa -out privkey.pem 2048` - privkey_pem = """-----BEGIN RSA PRIVATE KEY----- -MIIEowIBAAKCAQEAoa9i8v9YIzb+3yRHyXLm4j1eWK9lQc6lFwoQhik8y+joD+5A -v73OlDZAcn6vzlU72vwrJ1f4o54nEVm0rhNrhwCsiHCdxxEDEoqZ8w/19vc4hWj4 -SYwGirhcnyb2ysZSV8v9Lm5HiFe5zZM4jzCzf2rzt0YRlZZj9nhSglaiHZ9BE2e0 -vzOl6GePDz6yS4jbh2RsPsDQtqXNOqZwfGUd+iTsbSxXcm8+rNrT1VAbx6+1Sr0r -aDyc/jp8S1JwJ0ofJLsU3Pb6DYazFf12CNTsrKF1L0hAsbN8v2DSunZIQqQLQGfp -0hnNO9V8q9FjvVu8XY/HhgoTvtESU3vuq+BnIwIDAQABAoIBAGpWDP+/y9mtK8bZ -95SXyx10Ov6crD2xiIY0ilWR/XgmP6lqio8QaDK104D5rOpIyErnmgIQK2iAdTVG -CDyMbSWm3dIGLt5jY9/n5AQltSCtyzCCrvi/7PWC9vd9Csal1DYF5QeKY+VZvMtl -Tcduwj7EunEI1jvJYwkQbUNncsuDi+88/JNwa8DJp1IrR4goxNflGl7mNzfq49re -lhSyezfLSTZKDa3A6sYnNFAAOy82iXZuLXCqKuwRuaiFFilB0R0/egzBSUeBwMJk -sS+SvHHXwv9HsYt4pYiiZFm8HxB4NKYtdpHpvJVJcG9vOXjewnA5YHWVDJsrBfu6 -0kPgbcECgYEA0bqfX2Vc6DizwjWVn9yVlckjQNGTnwf/B9eGW2MgTn6YADe0yjFm -KCtr34hEZc/hv3kBnoLOqSvZJiser8ve3SmwxfmpjEfJdIgA5J5DbCEGBiDm9PMy -0lYsfjykzYykehdasb8f4xd+SPMuTC/CFb1MCTlohex7qn7Xt9IskBECgYEAxVtF -iXwFJPQUil2bSFGnxtaI/8ijypLOkP3CyuVnEcbMt74jDt1hdooRxjQ9VVlg7r7i -EvebPKMukWxdVcQ/38i97oB/oN7MIH0QBCDWTdTQokuNQSEknGLouj6YtLAWRcyJ -9DDENSaGtP42le5dD60hZc732jN09fGxNa6gN/MCgYB5ux98CGJ3q0mzBNUW17q/ -GOLsYXiUitidHZyveIas6M+i+LJn1WpdEG7pbLd+fL2kHEEzVutKx9efTtHd6bAu -oF8pWfLuKFCm4bXa/H1XyocrkXdcX7h0222xy9NAN0zUTK/okW2Zqu4yu2t47xNw -+NGkXPztFsjkugDNgiE5cQKBgQDDy/BqHPORnOIAACw9jF1SpKcYdPsiz5FGQawO -1ZbzCPMzW9y2M6YtD3/gzxUGZv0G/7OUs7h8aTybJBJZM7FXGHZud2ent0J2/Px1 -zAow/3DZgvEp63LCAFL5635ezM/cAbff3r3aKVW9nPOUvf3vvokC01oMTb68/kMc -ihoERwKBgFsoRUrgGPSfG1UZt8BpIXbG/8qfoy/Vy77BRqvJ6ZpdM9RPqdAl7Sih -cdqfxs8w0NVvj+gvM/1CGO0J9lZW2f1J81haIoyUpiITFdoyzLKXLhMSbaF4Y7Hn -yC/N5w3cCLa2LLKoLG8hagFDlXBGSmpT1zgKBk4YxNn6CLdMSzPR ------END RSA PRIVATE KEY----- -""" - - privkey = load_pem_private_key( - privkey_pem.encode("ascii"), password=None - ) + # Generated with `openssl genrsa -out openssl-rsa-2048-3.txt 2048` + pempath = DATA_PATH.child("openssl-rsa-2048-3.txt") + privkey = load_pem_private_key(pempath.getContent(), password=None) assert isinstance(privkey, PrivateKey) pubkey = privkey.public_key() assert isinstance(pubkey, PublicKey) @@ -827,39 +802,9 @@ def test_mkdir_with_children_and_known_private_key(alice): ] } - # Randomly generated with `openssl genrsa -out privkey.pem 2048` - privkey_pem = """-----BEGIN RSA PRIVATE KEY----- -MIIEowIBAAKCAQEA2PL5Ry2BGuuUtRJa20WS0fwBOqVIVSXDVuSvZFYTT1Xji19J -q+ohHcFnIIYHAq0zQG+NgNjK5rogY/5TfbwIhfwLufleeAdL9jXTfxan0o/wwFA1 -DAIHcYsTEYI2dfQe4acOLFY6/Hh6iXCbHvSzzUnEmYkgwCAZvc0v/lD8pMnz/6gQ -2nJnAASfFovcAvfr1T+MZzLJGQem3f2IFp1frurQyFmzFRtZMO5B9PDSsFG4yJVf -cz0iSP8wlc9QydImmJGRvu4xEOkx/55B/XaUdb6CIGpCTkLsDOlImvZt9UHDSgXq -qcE/T7SYMIXqbep64tJw9enjomH+n1KVh9UA2wIDAQABAoIBABCSTrQ/J5N010EV -i9cf810S0M03/tRyM/+ZLESPxp3Sw7TLrIbzNWBee5AibLqpnDaZzsc+yBDjusGo -lZwPFt+VJxgnki288PJ3nhYhFuSglhU6izLFnOfxZZ16wsozwYAfEJgWZh8O3N1O -uqqcqndN4TSRIu1KBm1XFQlqCkJT/stzYjO4k1vhgZT4pqhYRdx7q7FAap4v+sNs -Svhm1blvOXlyeumAbFBdGFttpTxIOGRzI1bp00jcLK4rgssTTxNyEiVu4oJhQY/k -0CptSUzpGio8DZ0/8bNnKCkw8YATUWJZQgSmKraRwAYMMR/SZa7WqjEc2KRTj6xQ -pHmYwZECgYEA700a/7ur8+EwTSulLgDveAOtTV0xEbhuq6cJQgNrEp2rbFqie6FX -g/YJKzEpEnUvj/yOzhEcw3CdQDUaxndlqY87QIhUWMcsnfMPsM1FjhmfksR8s3TF -WZNqa0RAKmcRoLohGclSvRV2OVU8+10mLUwJfR86Nl5+auR3LxWLyB8CgYEA6BaR -r+Z7oTlgkdEDVhnQ58Msktv58y28N+VIbYS79bV01jqUUlogm5uTvdvq5nyENXHx -gnK88mVzWYBMk83D01HlOC5DhpspTVEQQG2V/If6KZa56mxiHP3Mab9jLew9w/kA -g6l/04ATSA8g4i2H/Bz0eEyPEBt6o/+SO0Xv38UCgYEAyTTLvrrNmgF922UXPdcL -gp2U2bfBymSIqUuJPTgij0SDHlgWxlyieRImI2ryXdKqayav7BP3W10U2yfLm5RI -pokICPqX8Q2HNkdoqf/uu8xPn9gWAc3tIaQRlp+MVBrVd48IxeXA67tf7FT/MVrg -/rUwRUQ8bfqF0NrIW46COYECgYAYDJamGoT/DNoD4hutZVlvWpsY0LCS0U9qn1ik -+Jcde+MSe9l4uxwb48AocUxi+84bV6ZF9Su9FmQghxnoSu8ay6ar7qdSoGtkNp0v -f+uF0nVKr/Kt5vM3u9jdsFZPoOY5k2jJO9wiB2h4FBE9PqiTqFBw0sYUTjSkH8yA -VdvoXQKBgFqCC8Y82eVf0/ORGTgG/KhZ72WFQKHyAeryvoLuadZ6JAI6qW9U1l9P -18SMnCO+opGN5GH2Qx7gdg17KzWzTW1gnbv0QUPNnnYEJU8VYMelNuKa8tmNgFH7 -inAwsxbbWoR08ai4exzbJrNrLpDRg5ih2wMtknN6D8m+EAvBC/Gj ------END RSA PRIVATE KEY----- -""" - - privkey = load_pem_private_key( - privkey_pem.encode("ascii"), password=None - ) + # Generated with `openssl genrsa -out openssl-rsa-2048-4.txt 2048` + pempath = DATA_PATH.child("openssl-rsa-2048-4.txt") + privkey = load_pem_private_key(pempath.getContent(), password=None) assert isinstance(privkey, PrivateKey) pubkey = privkey.public_key() assert isinstance(pubkey, PublicKey) diff --git a/src/allmydata/test/data/openssl-rsa-2048-2.txt b/src/allmydata/test/data/openssl-rsa-2048-2.txt new file mode 100644 index 000000000..dd3174209 --- /dev/null +++ b/src/allmydata/test/data/openssl-rsa-2048-2.txt @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEowIBAAKCAQEAygMjLBKayDEioOZap2syJhUlqI7Dkk4zV5TfVxlQFO7bR410 +eJRJY1rHGIeZxQPjytsSJvqlYEJrvvVNdhi6XN/6NA3RFL6pDTHkYyM3qbrXqlYC +HUlkS2JAZzIFRizl6nG11yIbHjPsoG+vGSjGSzVIiOP4NeIssYLpoASTIppdZxy+ +syZ6zSmPhZu7W9X73aupLjFrIZpjeKfO2+GfUwEzAH0HckLIgJpQ+vK3sqbSik/2 +1oZK33M8uvtdmba7D3uJXmxWMTJ7oyFLDpDOMl7HSUv1lZY2O2qiDPYfGDUM1BRp +6blxE+BA2INr9NO4A4H8pzhikFnaFnkpH/AxowIDAQABAoIBABprXJ8386w42NmI +JtT8bPuUCm/H9AXfWlGa87aVZebG8kCiXFgktJBc3+ryWQbuIk12ZyJX52b2aNb5 +h97pDv50gGlsYSrAYKWMH91jTrVQ7UGmq/IelhJR0DBu10e9OXh21JxFJpzFl63H +zXOR5JUTa+ATSHPrl4LDp0A5OPDuWbBWa64yx7gUI9/tljbndplCrPjmIE6+h10M +sqxW5oJpLnZpWc73QQUTuPIr+A7fLgGJYHnyCFUu9OW4ZnxNEI3/wNHPvoxkYuHN +2qVonFESiAx9mBv7JzQ7X2KIB8doY3KL6S7sAKi/i/aP7EDJ9QEtl3BR3M8/XP8E +KJVORWECgYEA8Vbw75+aVMxHUl9BJc1zESxqVvr+R0NBqMO47CBj39sTJkXY37O3 +A7j4dzCorI0NaB7Jr+AI2ZZu9CaR31Y2mhAGbNLBPK8yn0Z7iWyDIqOW1OpMDs35 +h2CI1pFLjx1a3PzhsQdzZ68izWKYBdTs2scaFz/ntaPwwPEwORaMDZECgYEA1kie +YfMRJ2GwzvbR35WvEMhVxhnmA6yuRL15Pkb1WDR3iWGM0ld/u3N4sRVCx1nU4wk/ +MMqCRdm4JaxqzR/hl8+/sp3Aai15ecqR+F+ecwbbB2XKVHfi1nqClivYnB+GgCh1 +bQYUd9LT80sIQdBEW5MBdbMFnOkt+1sSpjf1wfMCgYBAavlyrIJQQhqDdSN5iKY/ +HkDgKKy4rs4W0u9IL7kY5mvtGlWyGFEwcC35+oX7UMcUVKt3A3C5S3sgNi9XkraO +VtqwL20e2pDDjNeqrcku9MVs3YEhrn79UJoV08B8WdSICgPf8eIu+cNrWPbFD7mN +B/oB3K/nfvPjPD2n70nA0QKBgGWJN3NWR9SPV8ZZ8gyt0qxzISGjd/hZxKHR3jeC +TBMlmVbBoIay61WZW6EdX+0yRcvmv8iQzLXoendvgZP8/VqAGGe8lEY7kgoB0LUO +Kfh7USHqO7tWq2fR2TrrP9KKpaLoiOvGK8CzZ7cq4Ji+5QU3XUO2NnypiR5Hg0i7 +z3m9AoGBAIEXtoSR9OTwdmrdIQn3vsaFOkN5pyYfvAvdeZ+7wwMg/ZOwhStwctbI +Um7XqocXU+8f/gjczgLgMJj+zqr+QDH5n4vSTUMPeN0gIugI9UwWnc2rhbRCgDdY +W6SwPQGDuGoUa5PxjggkyevUUmtXvGG9jnkt9kozQOA0lOF1vbw/ +-----END RSA PRIVATE KEY----- diff --git a/src/allmydata/test/data/openssl-rsa-2048-3.txt b/src/allmydata/test/data/openssl-rsa-2048-3.txt new file mode 100644 index 000000000..2c423dc1f --- /dev/null +++ b/src/allmydata/test/data/openssl-rsa-2048-3.txt @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEowIBAAKCAQEAoa9i8v9YIzb+3yRHyXLm4j1eWK9lQc6lFwoQhik8y+joD+5A +v73OlDZAcn6vzlU72vwrJ1f4o54nEVm0rhNrhwCsiHCdxxEDEoqZ8w/19vc4hWj4 +SYwGirhcnyb2ysZSV8v9Lm5HiFe5zZM4jzCzf2rzt0YRlZZj9nhSglaiHZ9BE2e0 +vzOl6GePDz6yS4jbh2RsPsDQtqXNOqZwfGUd+iTsbSxXcm8+rNrT1VAbx6+1Sr0r +aDyc/jp8S1JwJ0ofJLsU3Pb6DYazFf12CNTsrKF1L0hAsbN8v2DSunZIQqQLQGfp +0hnNO9V8q9FjvVu8XY/HhgoTvtESU3vuq+BnIwIDAQABAoIBAGpWDP+/y9mtK8bZ +95SXyx10Ov6crD2xiIY0ilWR/XgmP6lqio8QaDK104D5rOpIyErnmgIQK2iAdTVG +CDyMbSWm3dIGLt5jY9/n5AQltSCtyzCCrvi/7PWC9vd9Csal1DYF5QeKY+VZvMtl +Tcduwj7EunEI1jvJYwkQbUNncsuDi+88/JNwa8DJp1IrR4goxNflGl7mNzfq49re +lhSyezfLSTZKDa3A6sYnNFAAOy82iXZuLXCqKuwRuaiFFilB0R0/egzBSUeBwMJk +sS+SvHHXwv9HsYt4pYiiZFm8HxB4NKYtdpHpvJVJcG9vOXjewnA5YHWVDJsrBfu6 +0kPgbcECgYEA0bqfX2Vc6DizwjWVn9yVlckjQNGTnwf/B9eGW2MgTn6YADe0yjFm +KCtr34hEZc/hv3kBnoLOqSvZJiser8ve3SmwxfmpjEfJdIgA5J5DbCEGBiDm9PMy +0lYsfjykzYykehdasb8f4xd+SPMuTC/CFb1MCTlohex7qn7Xt9IskBECgYEAxVtF +iXwFJPQUil2bSFGnxtaI/8ijypLOkP3CyuVnEcbMt74jDt1hdooRxjQ9VVlg7r7i +EvebPKMukWxdVcQ/38i97oB/oN7MIH0QBCDWTdTQokuNQSEknGLouj6YtLAWRcyJ +9DDENSaGtP42le5dD60hZc732jN09fGxNa6gN/MCgYB5ux98CGJ3q0mzBNUW17q/ +GOLsYXiUitidHZyveIas6M+i+LJn1WpdEG7pbLd+fL2kHEEzVutKx9efTtHd6bAu +oF8pWfLuKFCm4bXa/H1XyocrkXdcX7h0222xy9NAN0zUTK/okW2Zqu4yu2t47xNw ++NGkXPztFsjkugDNgiE5cQKBgQDDy/BqHPORnOIAACw9jF1SpKcYdPsiz5FGQawO +1ZbzCPMzW9y2M6YtD3/gzxUGZv0G/7OUs7h8aTybJBJZM7FXGHZud2ent0J2/Px1 +zAow/3DZgvEp63LCAFL5635ezM/cAbff3r3aKVW9nPOUvf3vvokC01oMTb68/kMc +ihoERwKBgFsoRUrgGPSfG1UZt8BpIXbG/8qfoy/Vy77BRqvJ6ZpdM9RPqdAl7Sih +cdqfxs8w0NVvj+gvM/1CGO0J9lZW2f1J81haIoyUpiITFdoyzLKXLhMSbaF4Y7Hn +yC/N5w3cCLa2LLKoLG8hagFDlXBGSmpT1zgKBk4YxNn6CLdMSzPR +-----END RSA PRIVATE KEY----- diff --git a/src/allmydata/test/data/openssl-rsa-2048-4.txt b/src/allmydata/test/data/openssl-rsa-2048-4.txt new file mode 100644 index 000000000..534ae30bc --- /dev/null +++ b/src/allmydata/test/data/openssl-rsa-2048-4.txt @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEowIBAAKCAQEA2PL5Ry2BGuuUtRJa20WS0fwBOqVIVSXDVuSvZFYTT1Xji19J +q+ohHcFnIIYHAq0zQG+NgNjK5rogY/5TfbwIhfwLufleeAdL9jXTfxan0o/wwFA1 +DAIHcYsTEYI2dfQe4acOLFY6/Hh6iXCbHvSzzUnEmYkgwCAZvc0v/lD8pMnz/6gQ +2nJnAASfFovcAvfr1T+MZzLJGQem3f2IFp1frurQyFmzFRtZMO5B9PDSsFG4yJVf +cz0iSP8wlc9QydImmJGRvu4xEOkx/55B/XaUdb6CIGpCTkLsDOlImvZt9UHDSgXq +qcE/T7SYMIXqbep64tJw9enjomH+n1KVh9UA2wIDAQABAoIBABCSTrQ/J5N010EV +i9cf810S0M03/tRyM/+ZLESPxp3Sw7TLrIbzNWBee5AibLqpnDaZzsc+yBDjusGo +lZwPFt+VJxgnki288PJ3nhYhFuSglhU6izLFnOfxZZ16wsozwYAfEJgWZh8O3N1O +uqqcqndN4TSRIu1KBm1XFQlqCkJT/stzYjO4k1vhgZT4pqhYRdx7q7FAap4v+sNs +Svhm1blvOXlyeumAbFBdGFttpTxIOGRzI1bp00jcLK4rgssTTxNyEiVu4oJhQY/k +0CptSUzpGio8DZ0/8bNnKCkw8YATUWJZQgSmKraRwAYMMR/SZa7WqjEc2KRTj6xQ +pHmYwZECgYEA700a/7ur8+EwTSulLgDveAOtTV0xEbhuq6cJQgNrEp2rbFqie6FX +g/YJKzEpEnUvj/yOzhEcw3CdQDUaxndlqY87QIhUWMcsnfMPsM1FjhmfksR8s3TF +WZNqa0RAKmcRoLohGclSvRV2OVU8+10mLUwJfR86Nl5+auR3LxWLyB8CgYEA6BaR +r+Z7oTlgkdEDVhnQ58Msktv58y28N+VIbYS79bV01jqUUlogm5uTvdvq5nyENXHx +gnK88mVzWYBMk83D01HlOC5DhpspTVEQQG2V/If6KZa56mxiHP3Mab9jLew9w/kA +g6l/04ATSA8g4i2H/Bz0eEyPEBt6o/+SO0Xv38UCgYEAyTTLvrrNmgF922UXPdcL +gp2U2bfBymSIqUuJPTgij0SDHlgWxlyieRImI2ryXdKqayav7BP3W10U2yfLm5RI +pokICPqX8Q2HNkdoqf/uu8xPn9gWAc3tIaQRlp+MVBrVd48IxeXA67tf7FT/MVrg +/rUwRUQ8bfqF0NrIW46COYECgYAYDJamGoT/DNoD4hutZVlvWpsY0LCS0U9qn1ik ++Jcde+MSe9l4uxwb48AocUxi+84bV6ZF9Su9FmQghxnoSu8ay6ar7qdSoGtkNp0v +f+uF0nVKr/Kt5vM3u9jdsFZPoOY5k2jJO9wiB2h4FBE9PqiTqFBw0sYUTjSkH8yA +VdvoXQKBgFqCC8Y82eVf0/ORGTgG/KhZ72WFQKHyAeryvoLuadZ6JAI6qW9U1l9P +18SMnCO+opGN5GH2Qx7gdg17KzWzTW1gnbv0QUPNnnYEJU8VYMelNuKa8tmNgFH7 +inAwsxbbWoR08ai4exzbJrNrLpDRg5ih2wMtknN6D8m+EAvBC/Gj +-----END RSA PRIVATE KEY----- diff --git a/src/allmydata/test/test_dirnode.py b/src/allmydata/test/test_dirnode.py index 8cfc02a9a..93122ba19 100644 --- a/src/allmydata/test/test_dirnode.py +++ b/src/allmydata/test/test_dirnode.py @@ -9,6 +9,7 @@ from zope.interface import implementer from twisted.trial import unittest from twisted.internet import defer from twisted.internet.interfaces import IConsumer +from twisted.python.filepath import FilePath from allmydata import uri, dirnode from allmydata.client import _Client from allmydata.crypto.rsa import create_signing_keypair @@ -2035,38 +2036,9 @@ class DeterministicDirnode(testutil.ReallyEqualMixin, testutil.ShouldFailMixin, these values are derived deterministically, given the same keypair, the resulting filecap should also always be the same. """ - # Randomly generated with `openssl genrsa -out privkey.pem 2048` - privkey_pem = """-----BEGIN RSA PRIVATE KEY----- -MIIEowIBAAKCAQEAygMjLBKayDEioOZap2syJhUlqI7Dkk4zV5TfVxlQFO7bR410 -eJRJY1rHGIeZxQPjytsSJvqlYEJrvvVNdhi6XN/6NA3RFL6pDTHkYyM3qbrXqlYC -HUlkS2JAZzIFRizl6nG11yIbHjPsoG+vGSjGSzVIiOP4NeIssYLpoASTIppdZxy+ -syZ6zSmPhZu7W9X73aupLjFrIZpjeKfO2+GfUwEzAH0HckLIgJpQ+vK3sqbSik/2 -1oZK33M8uvtdmba7D3uJXmxWMTJ7oyFLDpDOMl7HSUv1lZY2O2qiDPYfGDUM1BRp -6blxE+BA2INr9NO4A4H8pzhikFnaFnkpH/AxowIDAQABAoIBABprXJ8386w42NmI -JtT8bPuUCm/H9AXfWlGa87aVZebG8kCiXFgktJBc3+ryWQbuIk12ZyJX52b2aNb5 -h97pDv50gGlsYSrAYKWMH91jTrVQ7UGmq/IelhJR0DBu10e9OXh21JxFJpzFl63H -zXOR5JUTa+ATSHPrl4LDp0A5OPDuWbBWa64yx7gUI9/tljbndplCrPjmIE6+h10M -sqxW5oJpLnZpWc73QQUTuPIr+A7fLgGJYHnyCFUu9OW4ZnxNEI3/wNHPvoxkYuHN -2qVonFESiAx9mBv7JzQ7X2KIB8doY3KL6S7sAKi/i/aP7EDJ9QEtl3BR3M8/XP8E -KJVORWECgYEA8Vbw75+aVMxHUl9BJc1zESxqVvr+R0NBqMO47CBj39sTJkXY37O3 -A7j4dzCorI0NaB7Jr+AI2ZZu9CaR31Y2mhAGbNLBPK8yn0Z7iWyDIqOW1OpMDs35 -h2CI1pFLjx1a3PzhsQdzZ68izWKYBdTs2scaFz/ntaPwwPEwORaMDZECgYEA1kie -YfMRJ2GwzvbR35WvEMhVxhnmA6yuRL15Pkb1WDR3iWGM0ld/u3N4sRVCx1nU4wk/ -MMqCRdm4JaxqzR/hl8+/sp3Aai15ecqR+F+ecwbbB2XKVHfi1nqClivYnB+GgCh1 -bQYUd9LT80sIQdBEW5MBdbMFnOkt+1sSpjf1wfMCgYBAavlyrIJQQhqDdSN5iKY/ -HkDgKKy4rs4W0u9IL7kY5mvtGlWyGFEwcC35+oX7UMcUVKt3A3C5S3sgNi9XkraO -VtqwL20e2pDDjNeqrcku9MVs3YEhrn79UJoV08B8WdSICgPf8eIu+cNrWPbFD7mN -B/oB3K/nfvPjPD2n70nA0QKBgGWJN3NWR9SPV8ZZ8gyt0qxzISGjd/hZxKHR3jeC -TBMlmVbBoIay61WZW6EdX+0yRcvmv8iQzLXoendvgZP8/VqAGGe8lEY7kgoB0LUO -Kfh7USHqO7tWq2fR2TrrP9KKpaLoiOvGK8CzZ7cq4Ji+5QU3XUO2NnypiR5Hg0i7 -z3m9AoGBAIEXtoSR9OTwdmrdIQn3vsaFOkN5pyYfvAvdeZ+7wwMg/ZOwhStwctbI -Um7XqocXU+8f/gjczgLgMJj+zqr+QDH5n4vSTUMPeN0gIugI9UwWnc2rhbRCgDdY -W6SwPQGDuGoUa5PxjggkyevUUmtXvGG9jnkt9kozQOA0lOF1vbw/ ------END RSA PRIVATE KEY----- -""" - privkey = load_pem_private_key( - privkey_pem.encode("ascii"), password=None - ) + # Generated with `openssl genrsa -out openssl-rsa-2048-2.txt 2048` + pempath = FilePath(__file__).sibling("data").child("openssl-rsa-2048-2.txt") + privkey = load_pem_private_key(pempath.getContent(), password=None) pubkey = privkey.public_key() writekey, _, fingerprint = derive_mutable_keys((pubkey, privkey)) From f694224aba0adec125786097d744a694fc72ec0f Mon Sep 17 00:00:00 2001 From: "Christopher R. Wood" Date: Thu, 30 May 2024 15:50:40 -0400 Subject: [PATCH 20/22] Add docstring for `create_dirnode` --- src/allmydata/client.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index 6edbf7eeb..e942efcc0 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -1132,6 +1132,32 @@ class _Client(node.Node, pollmixin.PollMixin): *, unique_keypair: tuple[rsa.PublicKey, rsa.PrivateKey] | None = None ): + """ + Create a new directory. + + :param initial_children: If given, a structured dict representing the + initial content of the created directory. See + `docs/frontends/webapi.rst` for examples. + + :param version: If given, an int representing the mutable file format + of the new object. Acceptable values are currently `SDMF_VERSION` + or `MDMF_VERSION` (corresponding to 0 or 1, respectively, as + defined in `allmydata.interfaces`). If no such value is provided, + the default mutable format will be used (currently SDMF). + + :param unique_keypair: an optional tuple containing the RSA public + and private key to be used for the new directory. Typically, this + value is omitted (in which case a new random keypair will be + generated at creation time). + + **Warning** This value independently determines the identity of + the mutable object to create. There cannot be two different + mutable objects that share a keypair. They will merge into one + object (with undefined contents). + + :return: A Deferred which will fire with a representation of the new + directory after it has been created. + """ d = self.nodemaker.create_new_mutable_directory( initial_children, version=version, From 00f82a46ae5fae5a706c75fc204a1f002dc13bce Mon Sep 17 00:00:00 2001 From: "Christopher R. Wood" Date: Thu, 30 May 2024 15:51:24 -0400 Subject: [PATCH 21/22] Expand type hints for `create_dirnode` --- src/allmydata/client.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/allmydata/client.py b/src/allmydata/client.py index e942efcc0..48f372b05 100644 --- a/src/allmydata/client.py +++ b/src/allmydata/client.py @@ -32,6 +32,7 @@ import allmydata from allmydata import node from allmydata.crypto import rsa, ed25519 from allmydata.crypto.util import remove_prefix +from allmydata.dirnode import DirectoryNode from allmydata.storage.server import StorageServer, FoolscapStorageServer from allmydata import storage_client from allmydata.immutable.upload import Uploader @@ -1127,11 +1128,11 @@ class _Client(node.Node, pollmixin.PollMixin): def create_dirnode( self, - initial_children=None, - version=None, + initial_children: dict | None = None, + version: int | None = None, *, unique_keypair: tuple[rsa.PublicKey, rsa.PrivateKey] | None = None - ): + ) -> DirectoryNode: """ Create a new directory. From 5f4d5de739f2524a21dc23511536092f148c4dd1 Mon Sep 17 00:00:00 2001 From: Florian Sesser Date: Thu, 20 Jun 2024 13:30:50 +0000 Subject: [PATCH 22/22] Move GBS specification ... from 'docs/proposed' to 'docs/specification'. Fixes ticket:4019 --- docs/proposed/index.rst | 1 - docs/{proposed => specifications}/http-storage-node-protocol.rst | 0 docs/specifications/index.rst | 1 + 3 files changed, 1 insertion(+), 1 deletion(-) rename docs/{proposed => specifications}/http-storage-node-protocol.rst (100%) diff --git a/docs/proposed/index.rst b/docs/proposed/index.rst index d01d92d2d..f0bb2f344 100644 --- a/docs/proposed/index.rst +++ b/docs/proposed/index.rst @@ -14,4 +14,3 @@ index only lists the files that are in .rst format. :maxdepth: 2 leasedb - http-storage-node-protocol diff --git a/docs/proposed/http-storage-node-protocol.rst b/docs/specifications/http-storage-node-protocol.rst similarity index 100% rename from docs/proposed/http-storage-node-protocol.rst rename to docs/specifications/http-storage-node-protocol.rst diff --git a/docs/specifications/index.rst b/docs/specifications/index.rst index e813acf07..4f71dc0dc 100644 --- a/docs/specifications/index.rst +++ b/docs/specifications/index.rst @@ -17,3 +17,4 @@ the data formats used by Tahoe. lease servers-of-happiness backends/raic + http-storage-node-protocol