Merge branch 'master' into remove-future--a-detiste

benchmarks/conftest.py

@@ -101,7 +101,7 @@ def client_node(request, grid, storage_nodes, number_of_nodes) -> Client:
             "client_node",
             needed=number_of_nodes,
             happy=number_of_nodes,
-            total=number_of_nodes,
+            total=number_of_nodes + 3,  # Make sure FEC does some work
         )
     )
     print(f"Client node pid: {client_node.process.transport.pid}")

docs/frontends/webapi.rst

@@ -446,6 +446,21 @@ Creating a New Directory
  given, the directory's format is determined by the default mutable file
  format, as configured on the Tahoe-LAFS node responding to the request.
 
+ In addition, an optional "private-key=" argument is supported which, if given,
+ specifies the underlying signing key to be used when creating the directory.
+ This value must be a DER-encoded 2048-bit RSA private key in urlsafe base64
+ encoding. (To convert an existing PEM-encoded RSA key file into the format
+ required, the following commands may be used -- assuming a modern UNIX-like
+ environment with common tools already installed:
+ ``openssl rsa -in key.pem -outform der | base64 -w 0 -i - | tr '+/' '-_'``)
+
+ Because this key can be used to derive the write capability for the
+ associated directory, additional care should be taken to ensure that the key is
+ unique, that it is kept confidential, and that it was derived from an
+ appropriate (high-entropy) source of randomness. If this argument is omitted
+ (the default behavior), Tahoe-LAFS will generate an appropriate signing key
+ using the underlying operating system's source of entropy.
+
 ``POST /uri?t=mkdir-with-children``
 
  Create a new directory, populated with a set of child nodes, and return its
@@ -453,7 +468,8 @@ Creating a New Directory
  any other directory: the returned write-cap is the only reference to it.
 
  The format of the directory can be controlled with the format= argument in
- the query string, as described above.
+ the query string and a signing key can be specified with the private-key=
+ argument, as described above.
 
  Initial children are provided as the body of the POST form (this is more
  efficient than doing separate mkdir and set_children operations). If the

docs/proposed/index.rst

@@ -14,4 +14,3 @@ index only lists the files that are in .rst format.
    :maxdepth: 2
 
    leasedb
-   http-storage-node-protocol

docs/specifications/index.rst

@@ -17,3 +17,4 @@ the data formats used by Tahoe.
    lease
    servers-of-happiness
    backends/raic
+   http-storage-node-protocol

integration/test_web.py

@@ -12,11 +12,21 @@ exists anywhere, however.
 from __future__ import annotations
 
 import time
+from base64 import urlsafe_b64encode
 from urllib.parse import unquote as url_unquote, quote as url_quote
 
+from cryptography.hazmat.primitives.serialization import load_pem_private_key
 from twisted.internet.threads import deferToThread
+from twisted.python.filepath import FilePath
 
 import allmydata.uri
+from allmydata.crypto.rsa import (
+    create_signing_keypair,
+    der_string_from_signing_key,
+    PrivateKey,
+    PublicKey,
+)
+from allmydata.mutable.common import derive_mutable_keys
 from allmydata.util import jsonbytes as json
 
 from . import util
@@ -28,6 +38,10 @@ from bs4 import BeautifulSoup
 
 import pytest_twisted
 
+
+DATA_PATH = FilePath(__file__).parent().sibling("src").child("allmydata").child("test").child("data")
+
+
 @run_in_thread
 def test_index(alice):
     """
@@ -541,3 +555,287 @@ def test_mkdir_with_children(alice):
     assert resp.startswith(b"URI:DIR2")
     cap = allmydata.uri.from_string(resp)
     assert isinstance(cap, allmydata.uri.DirectoryURI)
+
+
+@run_in_thread
+def test_mkdir_with_random_private_key(alice):
+    """
+    Create a new directory with ?t=mkdir&private-key=... using a
+    randomly-generated RSA private key.
+
+    The writekey and fingerprint derived from the provided RSA key
+    should match those of the newly-created directory capability.
+    """
+
+    privkey, pubkey = create_signing_keypair(2048)
+
+    writekey, _, fingerprint = derive_mutable_keys((pubkey, privkey))
+
+    # The "private-key" parameter takes a DER-encoded RSA private key
+    # encoded in URL-safe base64; PEM blocks are not supported.
+    privkey_der = der_string_from_signing_key(privkey)
+    privkey_encoded = urlsafe_b64encode(privkey_der).decode("ascii")
+
+    resp = util.web_post(
+        alice.process, u"uri",
+        params={
+            u"t": "mkdir",
+            u"private-key": privkey_encoded,
+        },
+    )
+    assert resp.startswith(b"URI:DIR2")
+
+    dircap = allmydata.uri.from_string(resp)
+    assert isinstance(dircap, allmydata.uri.DirectoryURI)
+
+    # DirectoryURI objects lack 'writekey' and 'fingerprint' attributes
+    # so extract them from the enclosed WriteableSSKFileURI object.
+    filecap = dircap.get_filenode_cap()
+    assert isinstance(filecap, allmydata.uri.WriteableSSKFileURI)
+
+    assert (writekey, fingerprint) == (filecap.writekey, filecap.fingerprint)
+
+
+@run_in_thread
+def test_mkdir_with_known_private_key(alice):
+    """
+    Create a new directory with ?t=mkdir&private-key=... using a
+    known-in-advance RSA private key.
+
+    The writekey and fingerprint derived from the provided RSA key
+    should match those of the newly-created directory capability.
+    In addition, because the writekey and fingerprint are derived
+    deterministically, given the same RSA private key, the resultant
+    directory capability should always be the same.
+    """
+    # Generated with `openssl genrsa -out openssl-rsa-2048-3.txt 2048`
+    pempath = DATA_PATH.child("openssl-rsa-2048-3.txt")
+    privkey = load_pem_private_key(pempath.getContent(), password=None)
+    assert isinstance(privkey, PrivateKey)
+    pubkey = privkey.public_key()
+    assert isinstance(pubkey, PublicKey)
+
+    writekey, _, fingerprint = derive_mutable_keys((pubkey, privkey))
+
+    # The "private-key" parameter takes a DER-encoded RSA private key
+    # encoded in URL-safe base64; PEM blocks are not supported.
+    privkey_der = der_string_from_signing_key(privkey)
+    privkey_encoded = urlsafe_b64encode(privkey_der).decode("ascii")
+
+    resp = util.web_post(
+        alice.process, u"uri",
+        params={
+            u"t": "mkdir",
+            u"private-key": privkey_encoded,
+        },
+    )
+    assert resp.startswith(b"URI:DIR2")
+
+    dircap = allmydata.uri.from_string(resp)
+    assert isinstance(dircap, allmydata.uri.DirectoryURI)
+
+    # DirectoryURI objects lack 'writekey' and 'fingerprint' attributes
+    # so extract them from the enclosed WriteableSSKFileURI object.
+    filecap = dircap.get_filenode_cap()
+    assert isinstance(filecap, allmydata.uri.WriteableSSKFileURI)
+
+    assert (writekey, fingerprint) == (filecap.writekey, filecap.fingerprint)
+
+    assert resp == b"URI:DIR2:3oo7j7f7qqxnet2z2lf57ucup4:cpktmsxlqnd5yeekytxjxvff5e6d6fv7py6rftugcndvss7tzd2a"
+
+
+@run_in_thread
+def test_mkdir_with_children_and_random_private_key(alice):
+    """
+    Create a new directory with ?t=mkdir-with-children&private-key=...
+    using a randomly-generated RSA private key.
+
+    The writekey and fingerprint derived from the provided RSA key
+    should match those of the newly-created directory capability.
+    """
+
+    # create a file to put in our directory
+    FILE_CONTENTS = u"some file contents\n" * 500
+    resp = requests.put(
+        util.node_url(alice.process.node_dir, u"uri"),
+        data=FILE_CONTENTS,
+    )
+    filecap = resp.content.strip()
+
+    # create a (sub) directory to put in our directory
+    resp = requests.post(
+        util.node_url(alice.process.node_dir, u"uri"),
+        params={
+            u"t": u"mkdir",
+        }
+    )
+    # (we need both the read-write and read-only URIs I guess)
+    dircap = resp.content
+    dircap_obj = allmydata.uri.from_string(dircap)
+    dircap_ro = dircap_obj.get_readonly().to_string()
+
+    # create json information about our directory
+    meta = {
+        "a_file": [
+            "filenode", {
+                "ro_uri": filecap,
+                "metadata": {
+                    "ctime": 1202777696.7564139,
+                    "mtime": 1202777696.7564139,
+                    "tahoe": {
+                        "linkcrtime": 1202777696.7564139,
+                        "linkmotime": 1202777696.7564139
+                    }
+                }
+            }
+        ],
+        "some_subdir": [
+            "dirnode", {
+                "rw_uri": dircap,
+                "ro_uri": dircap_ro,
+                "metadata": {
+                    "ctime": 1202778102.7589991,
+                    "mtime": 1202778111.2160511,
+                    "tahoe": {
+                        "linkcrtime": 1202777696.7564139,
+                        "linkmotime": 1202777696.7564139
+                    }
+                }
+            }
+        ]
+    }
+
+    privkey, pubkey = create_signing_keypair(2048)
+
+    writekey, _, fingerprint = derive_mutable_keys((pubkey, privkey))
+
+    # The "private-key" parameter takes a DER-encoded RSA private key
+    # encoded in URL-safe base64; PEM blocks are not supported.
+    privkey_der = der_string_from_signing_key(privkey)
+    privkey_encoded = urlsafe_b64encode(privkey_der).decode("ascii")
+
+    # create a new directory with one file and one sub-dir (all-at-once)
+    # with the supplied RSA private key
+    resp = util.web_post(
+        alice.process, u"uri",
+        params={
+            u"t": "mkdir-with-children",
+            u"private-key": privkey_encoded,
+        },
+        data=json.dumps(meta),
+    )
+    assert resp.startswith(b"URI:DIR2")
+
+    dircap = allmydata.uri.from_string(resp)
+    assert isinstance(dircap, allmydata.uri.DirectoryURI)
+
+    # DirectoryURI objects lack 'writekey' and 'fingerprint' attributes
+    # so extract them from the enclosed WriteableSSKFileURI object.
+    filecap = dircap.get_filenode_cap()
+    assert isinstance(filecap, allmydata.uri.WriteableSSKFileURI)
+
+    assert (writekey, fingerprint) == (filecap.writekey, filecap.fingerprint)
+
+
+@run_in_thread
+def test_mkdir_with_children_and_known_private_key(alice):
+    """
+    Create a new directory with ?t=mkdir-with-children&private-key=...
+    using a known-in-advance RSA private key.
+
+
+    The writekey and fingerprint derived from the provided RSA key
+    should match those of the newly-created directory capability.
+    In addition, because the writekey and fingerprint are derived
+    deterministically, given the same RSA private key, the resultant
+    directory capability should always be the same.
+    """
+
+    # create a file to put in our directory
+    FILE_CONTENTS = u"some file contents\n" * 500
+    resp = requests.put(
+        util.node_url(alice.process.node_dir, u"uri"),
+        data=FILE_CONTENTS,
+    )
+    filecap = resp.content.strip()
+
+    # create a (sub) directory to put in our directory
+    resp = requests.post(
+        util.node_url(alice.process.node_dir, u"uri"),
+        params={
+            u"t": u"mkdir",
+        }
+    )
+    # (we need both the read-write and read-only URIs I guess)
+    dircap = resp.content
+    dircap_obj = allmydata.uri.from_string(dircap)
+    dircap_ro = dircap_obj.get_readonly().to_string()
+
+    # create json information about our directory
+    meta = {
+        "a_file": [
+            "filenode", {
+                "ro_uri": filecap,
+                "metadata": {
+                    "ctime": 1202777696.7564139,
+                    "mtime": 1202777696.7564139,
+                    "tahoe": {
+                        "linkcrtime": 1202777696.7564139,
+                        "linkmotime": 1202777696.7564139
+                    }
+                }
+            }
+        ],
+        "some_subdir": [
+            "dirnode", {
+                "rw_uri": dircap,
+                "ro_uri": dircap_ro,
+                "metadata": {
+                    "ctime": 1202778102.7589991,
+                    "mtime": 1202778111.2160511,
+                    "tahoe": {
+                        "linkcrtime": 1202777696.7564139,
+                        "linkmotime": 1202777696.7564139
+                    }
+                }
+            }
+        ]
+    }
+
+    # Generated with `openssl genrsa -out openssl-rsa-2048-4.txt 2048`
+    pempath = DATA_PATH.child("openssl-rsa-2048-4.txt")
+    privkey = load_pem_private_key(pempath.getContent(), password=None)
+    assert isinstance(privkey, PrivateKey)
+    pubkey = privkey.public_key()
+    assert isinstance(pubkey, PublicKey)
+
+    writekey, _, fingerprint = derive_mutable_keys((pubkey, privkey))
+
+    # The "private-key" parameter takes a DER-encoded RSA private key
+    # encoded in URL-safe base64; PEM blocks are not supported.
+    privkey_der = der_string_from_signing_key(privkey)
+    privkey_encoded = urlsafe_b64encode(privkey_der).decode("ascii")
+
+    # create a new directory with one file and one sub-dir (all-at-once)
+    # with the supplied RSA private key
+    resp = util.web_post(
+        alice.process, u"uri",
+        params={
+            u"t": "mkdir-with-children",
+            u"private-key": privkey_encoded,
+        },
+        data=json.dumps(meta),
+    )
+    assert resp.startswith(b"URI:DIR2")
+
+    dircap = allmydata.uri.from_string(resp)
+    assert isinstance(dircap, allmydata.uri.DirectoryURI)
+
+    # DirectoryURI objects lack 'writekey' and 'fingerprint' attributes
+    # so extract them from the enclosed WriteableSSKFileURI object.
+    filecap = dircap.get_filenode_cap()
+    assert isinstance(filecap, allmydata.uri.WriteableSSKFileURI)
+
+    assert (writekey, fingerprint) == (filecap.writekey, filecap.fingerprint)
+
+    assert resp == b"URI:DIR2:ppwzpwrd37xi7tpribxyaa25uy:imdws47wwpzfkc5vfllo4ugspb36iit4cqps6ttuhaouc66jb2da"

newsfragments/4072.feature

@@ -0,0 +1 @@
+Continued work to make Tahoe-LAFS take advantage of multiple CPUs.

newsfragments/4094.feature

@@ -0,0 +1 @@
+Mutable directories can now be created with a pre-determined "signature key" via the web API using the "private-key=..." parameter. The "private-key" value must be a DER-encoded 2048-bit RSA private key in urlsafe base64 encoding.

src/allmydata/client.py

@@ -32,6 +32,7 @@ import allmydata
 from allmydata import node
 from allmydata.crypto import rsa, ed25519
 from allmydata.crypto.util import remove_prefix
+from allmydata.dirnode import DirectoryNode
 from allmydata.storage.server import StorageServer, FoolscapStorageServer
 from allmydata import storage_client
 from allmydata.immutable.upload import Uploader
@@ -1125,8 +1126,44 @@ class _Client(node.Node, pollmixin.PollMixin):
         # may get an opaque node if there were any problems.
         return self.nodemaker.create_from_cap(write_uri, read_uri, deep_immutable=deep_immutable, name=name)
 
-    def create_dirnode(self, initial_children=None, version=None):
+    def create_dirnode(
-        d = self.nodemaker.create_new_mutable_directory(initial_children, version=version)
+        self,
+        initial_children: dict | None = None,
+        version: int | None = None,
+        *,
+        unique_keypair: tuple[rsa.PublicKey, rsa.PrivateKey] | None = None
+    ) -> DirectoryNode:
+        """
+        Create a new directory.
+
+        :param initial_children: If given, a structured dict representing the
+            initial content of the created directory. See
+            `docs/frontends/webapi.rst` for examples.
+
+        :param version: If given, an int representing the mutable file format
+            of the new object. Acceptable values are currently `SDMF_VERSION`
+            or `MDMF_VERSION` (corresponding to 0 or 1, respectively, as
+            defined in `allmydata.interfaces`). If no such value is provided,
+            the default mutable format will be used (currently SDMF).
+
+        :param unique_keypair: an optional tuple containing the RSA public
+            and private key to be used for the new directory. Typically, this
+            value is omitted (in which case a new random keypair will be
+            generated at creation time).
+
+            **Warning** This value independently determines the identity of
+            the mutable object to create.  There cannot be two different
+            mutable objects that share a keypair.  They will merge into one
+            object (with undefined contents).
+
+        :return: A Deferred which will fire with a representation of the new
+            directory after it has been created.
+        """
+        d = self.nodemaker.create_new_mutable_directory(
+            initial_children,
+            version=version,
+            keypair=unique_keypair,
+        )
         return d
 
     def create_immutable_dirnode(self, children, convergence=None):

src/allmydata/crypto/aes.py

@@ -77,8 +77,8 @@ def encrypt_data(encryptor, plaintext):
     """
 
     _validate_cryptor(encryptor, encrypt=True)
-    if not isinstance(plaintext, bytes):
+    if not isinstance(plaintext, (bytes, memoryview)):
-        raise ValueError('Plaintext must be bytes')
+        raise ValueError(f'Plaintext must be bytes or memoryview: {type(plaintext)}')
 
     return encryptor.update(plaintext)
 
@@ -116,8 +116,8 @@ def decrypt_data(decryptor, plaintext):
     """
 
     _validate_cryptor(decryptor, encrypt=False)
-    if not isinstance(plaintext, bytes):
+    if not isinstance(plaintext, (bytes, memoryview)):
-        raise ValueError('Plaintext must be bytes')
+        raise ValueError(f'Plaintext must be bytes or memoryview: {type(plaintext)}')
 
     return decryptor.update(plaintext)
 

src/allmydata/immutable/downloader/node.py

@@ -411,7 +411,7 @@ class DownloadNode(object):
 
     def process_blocks(self, segnum, blocks):
         start = now()
-        d = defer.maybeDeferred(self._decode_blocks, segnum, blocks)
+        d = self._decode_blocks(segnum, blocks)
         d.addCallback(self._check_ciphertext_hash, segnum)
         def _deliver(result):
             log.msg(format="delivering segment(%(segnum)d)",

src/allmydata/mutable/filenode.py

@@ -14,6 +14,7 @@ from allmydata.interfaces import IMutableFileNode, ICheckable, ICheckResults, \
      IMutableFileVersion, IWriteable
 from allmydata.util import hashutil, log, consumer, deferredutil, mathutil
 from allmydata.util.assertutil import precondition
+from allmydata.util.cputhreadpool import defer_to_thread
 from allmydata.uri import WriteableSSKFileURI, ReadonlySSKFileURI, \
                           WriteableMDMFFileURI, ReadonlyMDMFFileURI
 from allmydata.monitor import Monitor
@@ -128,7 +129,8 @@ class MutableFileNode(object):
 
         return self
 
-    def create_with_keys(self, keypair, contents,
+    @deferredutil.async_to_deferred
+    async def create_with_keys(self, keypair, contents,
                          version=SDMF_VERSION):
         """Call this to create a brand-new mutable file. It will create the
         shares, find homes for them, and upload the initial contents (created
@@ -137,8 +139,8 @@ class MutableFileNode(object):
         use) when it completes.
         """
         self._pubkey, self._privkey = keypair
-        self._writekey, self._encprivkey, self._fingerprint = derive_mutable_keys(
+        self._writekey, self._encprivkey, self._fingerprint = await defer_to_thread(
-            keypair,
+            derive_mutable_keys, keypair
         )
         if version == MDMF_VERSION:
             self._uri = WriteableMDMFFileURI(self._writekey, self._fingerprint)
@@ -149,7 +151,7 @@ class MutableFileNode(object):
         self._readkey = self._uri.readkey
         self._storage_index = self._uri.storage_index
         initial_contents = self._get_initial_contents(contents)
-        return self._upload(initial_contents, None)
+        return await self._upload(initial_contents, None)
 
     def _get_initial_contents(self, contents):
         if contents is None:

src/allmydata/mutable/retrieve.py

@@ -4,8 +4,8 @@ Ported to Python 3.
 from __future__ import annotations
 
 import time
-
 from itertools import count
+
 from zope.interface import implementer
 from twisted.internet import defer
 from twisted.python import failure
@@ -873,11 +873,20 @@ class Retrieve(object):
         shares = shares[:self._required_shares]
         self.log("decoding segment %d" % segnum)
         if segnum == self._num_segments - 1:
-            d = defer.maybeDeferred(self._tail_decoder.decode, shares, shareids)
+            d = self._tail_decoder.decode(shares, shareids)
         else:
-            d = defer.maybeDeferred(self._segment_decoder.decode, shares, shareids)
+            d = self._segment_decoder.decode(shares, shareids)
-        def _process(buffers):
+
-            segment = b"".join(buffers)
+        # For larger shares, this can take a few milliseconds. As such, we want
+        # to unblock the event loop. In newer Python b"".join() will release
+        # the GIL: https://github.com/python/cpython/issues/80232
+        @deferredutil.async_to_deferred
+        async def _got_buffers(buffers):
+            return await defer_to_thread(lambda: b"".join(buffers))
+
+        d.addCallback(_got_buffers)
+
+        def _process(segment):
             self.log(format="now decoding segment %(segnum)s of %(numsegs)s",
                      segnum=segnum,
                      numsegs=self._num_segments,
@@ -928,12 +937,20 @@ class Retrieve(object):
             reason,
         )
 
-
+    @deferredutil.async_to_deferred
-    def _try_to_validate_privkey(self, enc_privkey, reader, server):
+    async def _try_to_validate_privkey(self, enc_privkey, reader, server):
         node_writekey = self._node.get_writekey()
-        alleged_privkey_s = decrypt_privkey(node_writekey, enc_privkey)
+
-        alleged_writekey = hashutil.ssk_writekey_hash(alleged_privkey_s)
+        def get_privkey():
-        if alleged_writekey != node_writekey:
+            alleged_privkey_s = decrypt_privkey(node_writekey, enc_privkey)
+            alleged_writekey = hashutil.ssk_writekey_hash(alleged_privkey_s)
+            if alleged_writekey != node_writekey:
+                return None
+            privkey, _ = rsa.create_signing_keypair_from_string(alleged_privkey_s)
+            return privkey
+
+        privkey = await defer_to_thread(get_privkey)
+        if privkey is None:
             self.log("invalid privkey from %s shnum %d" %
                      (reader, reader.shnum),
                      level=log.WEIRD, umid="YIw4tA")
@@ -950,7 +967,6 @@ class Retrieve(object):
         # it's good
         self.log("got valid privkey from shnum %d on reader %s" %
                  (reader.shnum, reader))
-        privkey, _ = rsa.create_signing_keypair_from_string(alleged_privkey_s)
         self._node._populate_encprivkey(enc_privkey)
         self._node._populate_privkey(privkey)
         self._need_privkey = False

src/allmydata/nodemaker.py

@@ -135,7 +135,13 @@ class NodeMaker(object):
         d.addCallback(lambda res: n)
         return d
 
-    def create_new_mutable_directory(self, initial_children=None, version=None):
+    def create_new_mutable_directory(
+        self,
+        initial_children=None,
+        version=None,
+        *,
+        keypair: tuple[PublicKey, PrivateKey] | None = None,
+    ):
         if initial_children is None:
             initial_children = {}
         for (name, (node, metadata)) in initial_children.items():
@@ -145,7 +151,8 @@ class NodeMaker(object):
         d = self.create_mutable_file(lambda n:
                                      MutableData(pack_children(initial_children,
                                                     n.get_writekey())),
-                                     version=version)
+                                     version=version,
+                                     keypair=keypair)
         d.addCallback(self._create_dirnode)
         return d
 

src/allmydata/test/data/openssl-rsa-2048-2.txt

@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEowIBAAKCAQEAygMjLBKayDEioOZap2syJhUlqI7Dkk4zV5TfVxlQFO7bR410
+eJRJY1rHGIeZxQPjytsSJvqlYEJrvvVNdhi6XN/6NA3RFL6pDTHkYyM3qbrXqlYC
+HUlkS2JAZzIFRizl6nG11yIbHjPsoG+vGSjGSzVIiOP4NeIssYLpoASTIppdZxy+
+syZ6zSmPhZu7W9X73aupLjFrIZpjeKfO2+GfUwEzAH0HckLIgJpQ+vK3sqbSik/2
+1oZK33M8uvtdmba7D3uJXmxWMTJ7oyFLDpDOMl7HSUv1lZY2O2qiDPYfGDUM1BRp
+6blxE+BA2INr9NO4A4H8pzhikFnaFnkpH/AxowIDAQABAoIBABprXJ8386w42NmI
+JtT8bPuUCm/H9AXfWlGa87aVZebG8kCiXFgktJBc3+ryWQbuIk12ZyJX52b2aNb5
+h97pDv50gGlsYSrAYKWMH91jTrVQ7UGmq/IelhJR0DBu10e9OXh21JxFJpzFl63H
+zXOR5JUTa+ATSHPrl4LDp0A5OPDuWbBWa64yx7gUI9/tljbndplCrPjmIE6+h10M
+sqxW5oJpLnZpWc73QQUTuPIr+A7fLgGJYHnyCFUu9OW4ZnxNEI3/wNHPvoxkYuHN
+2qVonFESiAx9mBv7JzQ7X2KIB8doY3KL6S7sAKi/i/aP7EDJ9QEtl3BR3M8/XP8E
+KJVORWECgYEA8Vbw75+aVMxHUl9BJc1zESxqVvr+R0NBqMO47CBj39sTJkXY37O3
+A7j4dzCorI0NaB7Jr+AI2ZZu9CaR31Y2mhAGbNLBPK8yn0Z7iWyDIqOW1OpMDs35
+h2CI1pFLjx1a3PzhsQdzZ68izWKYBdTs2scaFz/ntaPwwPEwORaMDZECgYEA1kie
+YfMRJ2GwzvbR35WvEMhVxhnmA6yuRL15Pkb1WDR3iWGM0ld/u3N4sRVCx1nU4wk/
+MMqCRdm4JaxqzR/hl8+/sp3Aai15ecqR+F+ecwbbB2XKVHfi1nqClivYnB+GgCh1
+bQYUd9LT80sIQdBEW5MBdbMFnOkt+1sSpjf1wfMCgYBAavlyrIJQQhqDdSN5iKY/
+HkDgKKy4rs4W0u9IL7kY5mvtGlWyGFEwcC35+oX7UMcUVKt3A3C5S3sgNi9XkraO
+VtqwL20e2pDDjNeqrcku9MVs3YEhrn79UJoV08B8WdSICgPf8eIu+cNrWPbFD7mN
+B/oB3K/nfvPjPD2n70nA0QKBgGWJN3NWR9SPV8ZZ8gyt0qxzISGjd/hZxKHR3jeC
+TBMlmVbBoIay61WZW6EdX+0yRcvmv8iQzLXoendvgZP8/VqAGGe8lEY7kgoB0LUO
+Kfh7USHqO7tWq2fR2TrrP9KKpaLoiOvGK8CzZ7cq4Ji+5QU3XUO2NnypiR5Hg0i7
+z3m9AoGBAIEXtoSR9OTwdmrdIQn3vsaFOkN5pyYfvAvdeZ+7wwMg/ZOwhStwctbI
+Um7XqocXU+8f/gjczgLgMJj+zqr+QDH5n4vSTUMPeN0gIugI9UwWnc2rhbRCgDdY
+W6SwPQGDuGoUa5PxjggkyevUUmtXvGG9jnkt9kozQOA0lOF1vbw/
+-----END RSA PRIVATE KEY-----

src/allmydata/test/data/openssl-rsa-2048-3.txt

@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEowIBAAKCAQEAoa9i8v9YIzb+3yRHyXLm4j1eWK9lQc6lFwoQhik8y+joD+5A
+v73OlDZAcn6vzlU72vwrJ1f4o54nEVm0rhNrhwCsiHCdxxEDEoqZ8w/19vc4hWj4
+SYwGirhcnyb2ysZSV8v9Lm5HiFe5zZM4jzCzf2rzt0YRlZZj9nhSglaiHZ9BE2e0
+vzOl6GePDz6yS4jbh2RsPsDQtqXNOqZwfGUd+iTsbSxXcm8+rNrT1VAbx6+1Sr0r
+aDyc/jp8S1JwJ0ofJLsU3Pb6DYazFf12CNTsrKF1L0hAsbN8v2DSunZIQqQLQGfp
+0hnNO9V8q9FjvVu8XY/HhgoTvtESU3vuq+BnIwIDAQABAoIBAGpWDP+/y9mtK8bZ
+95SXyx10Ov6crD2xiIY0ilWR/XgmP6lqio8QaDK104D5rOpIyErnmgIQK2iAdTVG
+CDyMbSWm3dIGLt5jY9/n5AQltSCtyzCCrvi/7PWC9vd9Csal1DYF5QeKY+VZvMtl
+Tcduwj7EunEI1jvJYwkQbUNncsuDi+88/JNwa8DJp1IrR4goxNflGl7mNzfq49re
+lhSyezfLSTZKDa3A6sYnNFAAOy82iXZuLXCqKuwRuaiFFilB0R0/egzBSUeBwMJk
+sS+SvHHXwv9HsYt4pYiiZFm8HxB4NKYtdpHpvJVJcG9vOXjewnA5YHWVDJsrBfu6
+0kPgbcECgYEA0bqfX2Vc6DizwjWVn9yVlckjQNGTnwf/B9eGW2MgTn6YADe0yjFm
+KCtr34hEZc/hv3kBnoLOqSvZJiser8ve3SmwxfmpjEfJdIgA5J5DbCEGBiDm9PMy
+0lYsfjykzYykehdasb8f4xd+SPMuTC/CFb1MCTlohex7qn7Xt9IskBECgYEAxVtF
+iXwFJPQUil2bSFGnxtaI/8ijypLOkP3CyuVnEcbMt74jDt1hdooRxjQ9VVlg7r7i
+EvebPKMukWxdVcQ/38i97oB/oN7MIH0QBCDWTdTQokuNQSEknGLouj6YtLAWRcyJ
+9DDENSaGtP42le5dD60hZc732jN09fGxNa6gN/MCgYB5ux98CGJ3q0mzBNUW17q/
+GOLsYXiUitidHZyveIas6M+i+LJn1WpdEG7pbLd+fL2kHEEzVutKx9efTtHd6bAu
+oF8pWfLuKFCm4bXa/H1XyocrkXdcX7h0222xy9NAN0zUTK/okW2Zqu4yu2t47xNw
++NGkXPztFsjkugDNgiE5cQKBgQDDy/BqHPORnOIAACw9jF1SpKcYdPsiz5FGQawO
+1ZbzCPMzW9y2M6YtD3/gzxUGZv0G/7OUs7h8aTybJBJZM7FXGHZud2ent0J2/Px1
+zAow/3DZgvEp63LCAFL5635ezM/cAbff3r3aKVW9nPOUvf3vvokC01oMTb68/kMc
+ihoERwKBgFsoRUrgGPSfG1UZt8BpIXbG/8qfoy/Vy77BRqvJ6ZpdM9RPqdAl7Sih
+cdqfxs8w0NVvj+gvM/1CGO0J9lZW2f1J81haIoyUpiITFdoyzLKXLhMSbaF4Y7Hn
+yC/N5w3cCLa2LLKoLG8hagFDlXBGSmpT1zgKBk4YxNn6CLdMSzPR
+-----END RSA PRIVATE KEY-----

src/allmydata/test/data/openssl-rsa-2048-4.txt

@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEowIBAAKCAQEA2PL5Ry2BGuuUtRJa20WS0fwBOqVIVSXDVuSvZFYTT1Xji19J
+q+ohHcFnIIYHAq0zQG+NgNjK5rogY/5TfbwIhfwLufleeAdL9jXTfxan0o/wwFA1
+DAIHcYsTEYI2dfQe4acOLFY6/Hh6iXCbHvSzzUnEmYkgwCAZvc0v/lD8pMnz/6gQ
+2nJnAASfFovcAvfr1T+MZzLJGQem3f2IFp1frurQyFmzFRtZMO5B9PDSsFG4yJVf
+cz0iSP8wlc9QydImmJGRvu4xEOkx/55B/XaUdb6CIGpCTkLsDOlImvZt9UHDSgXq
+qcE/T7SYMIXqbep64tJw9enjomH+n1KVh9UA2wIDAQABAoIBABCSTrQ/J5N010EV
+i9cf810S0M03/tRyM/+ZLESPxp3Sw7TLrIbzNWBee5AibLqpnDaZzsc+yBDjusGo
+lZwPFt+VJxgnki288PJ3nhYhFuSglhU6izLFnOfxZZ16wsozwYAfEJgWZh8O3N1O
+uqqcqndN4TSRIu1KBm1XFQlqCkJT/stzYjO4k1vhgZT4pqhYRdx7q7FAap4v+sNs
+Svhm1blvOXlyeumAbFBdGFttpTxIOGRzI1bp00jcLK4rgssTTxNyEiVu4oJhQY/k
+0CptSUzpGio8DZ0/8bNnKCkw8YATUWJZQgSmKraRwAYMMR/SZa7WqjEc2KRTj6xQ
+pHmYwZECgYEA700a/7ur8+EwTSulLgDveAOtTV0xEbhuq6cJQgNrEp2rbFqie6FX
+g/YJKzEpEnUvj/yOzhEcw3CdQDUaxndlqY87QIhUWMcsnfMPsM1FjhmfksR8s3TF
+WZNqa0RAKmcRoLohGclSvRV2OVU8+10mLUwJfR86Nl5+auR3LxWLyB8CgYEA6BaR
+r+Z7oTlgkdEDVhnQ58Msktv58y28N+VIbYS79bV01jqUUlogm5uTvdvq5nyENXHx
+gnK88mVzWYBMk83D01HlOC5DhpspTVEQQG2V/If6KZa56mxiHP3Mab9jLew9w/kA
+g6l/04ATSA8g4i2H/Bz0eEyPEBt6o/+SO0Xv38UCgYEAyTTLvrrNmgF922UXPdcL
+gp2U2bfBymSIqUuJPTgij0SDHlgWxlyieRImI2ryXdKqayav7BP3W10U2yfLm5RI
+pokICPqX8Q2HNkdoqf/uu8xPn9gWAc3tIaQRlp+MVBrVd48IxeXA67tf7FT/MVrg
+/rUwRUQ8bfqF0NrIW46COYECgYAYDJamGoT/DNoD4hutZVlvWpsY0LCS0U9qn1ik
++Jcde+MSe9l4uxwb48AocUxi+84bV6ZF9Su9FmQghxnoSu8ay6ar7qdSoGtkNp0v
+f+uF0nVKr/Kt5vM3u9jdsFZPoOY5k2jJO9wiB2h4FBE9PqiTqFBw0sYUTjSkH8yA
+VdvoXQKBgFqCC8Y82eVf0/ORGTgG/KhZ72WFQKHyAeryvoLuadZ6JAI6qW9U1l9P
+18SMnCO+opGN5GH2Qx7gdg17KzWzTW1gnbv0QUPNnnYEJU8VYMelNuKa8tmNgFH7
+inAwsxbbWoR08ai4exzbJrNrLpDRg5ih2wMtknN6D8m+EAvBC/Gj
+-----END RSA PRIVATE KEY-----

src/allmydata/test/test_dirnode.py

@@ -9,8 +9,10 @@ from zope.interface import implementer
 from twisted.trial import unittest
 from twisted.internet import defer
 from twisted.internet.interfaces import IConsumer
+from twisted.python.filepath import FilePath
 from allmydata import uri, dirnode
 from allmydata.client import _Client
+from allmydata.crypto.rsa import create_signing_keypair
 from allmydata.immutable import upload
 from allmydata.immutable.literal import LiteralFileNode
 from allmydata.interfaces import IImmutableFileNode, IMutableFileNode, \
@@ -19,16 +21,25 @@ from allmydata.interfaces import IImmutableFileNode, IMutableFileNode, \
      IDeepCheckResults, IDeepCheckAndRepairResults, \
      MDMF_VERSION, SDMF_VERSION
 from allmydata.mutable.filenode import MutableFileNode
-from allmydata.mutable.common import UncoordinatedWriteError
+from allmydata.mutable.common import (
+    UncoordinatedWriteError,
+    derive_mutable_keys,
+)
 from allmydata.util import hashutil, base32
 from allmydata.util.netstring import split_netstring
 from allmydata.monitor import Monitor
 from allmydata.test.common import make_chk_file_uri, make_mutable_file_uri, \
      ErrorMixin
+from allmydata.test.mutable.util import (
+    FakeStorage,
+    make_nodemaker_with_peers,
+    make_peer,
+)
 from allmydata.test.no_network import GridTestMixin
 from allmydata.unknown import UnknownNode, strip_prefix_for_ro
 from allmydata.nodemaker import NodeMaker
 from base64 import b32decode
+from cryptography.hazmat.primitives.serialization import load_pem_private_key
 import allmydata.test.common_util as testutil
 
 from hypothesis import given
@@ -1978,3 +1989,75 @@ class Adder(GridTestMixin, unittest.TestCase, testutil.ShouldFailMixin):
 
         d.addCallback(_test_adder)
         return d
+
+
+class DeterministicDirnode(testutil.ReallyEqualMixin, testutil.ShouldFailMixin, unittest.TestCase):
+    def setUp(self):
+        # Copied from allmydata.test.mutable.test_filenode
+        super(DeterministicDirnode, self).setUp()
+        self._storage = FakeStorage()
+        self._peers = list(
+            make_peer(self._storage, n)
+            for n
+            in range(10)
+        )
+        self.nodemaker = make_nodemaker_with_peers(self._peers)
+
+    async def test_create_with_random_keypair(self):
+        """
+        Create a dirnode using a random RSA keypair.
+
+        The writekey and fingerprint of the enclosed mutable filecap
+        should match those derived from the given keypair.
+        """
+        privkey, pubkey = create_signing_keypair(2048)
+        writekey, _, fingerprint = derive_mutable_keys((pubkey, privkey))
+
+        node = await self.nodemaker.create_new_mutable_directory(
+            keypair=(pubkey, privkey)
+        )
+        self.failUnless(isinstance(node, dirnode.DirectoryNode))
+
+        dircap = uri.from_string(node.get_uri())
+        self.failUnless(isinstance(dircap, uri.DirectoryURI))
+
+        filecap = dircap.get_filenode_cap()
+        self.failUnless(isinstance(filecap, uri.WriteableSSKFileURI))
+
+        self.failUnlessReallyEqual(filecap.writekey, writekey)
+        self.failUnlessReallyEqual(filecap.fingerprint, fingerprint)
+
+    async def test_create_with_known_keypair(self):
+        """
+        Create a dirnode using a known RSA keypair.
+
+        The writekey and fingerprint of the enclosed mutable filecap
+        should match those derived from the given keypair. Because
+        these values are derived deterministically, given the same
+        keypair, the resulting filecap should also always be the same.
+        """
+        # Generated with `openssl genrsa -out openssl-rsa-2048-2.txt 2048`
+        pempath = FilePath(__file__).sibling("data").child("openssl-rsa-2048-2.txt")
+        privkey = load_pem_private_key(pempath.getContent(), password=None)
+        pubkey = privkey.public_key()
+        writekey, _, fingerprint = derive_mutable_keys((pubkey, privkey))
+
+        node = await self.nodemaker.create_new_mutable_directory(
+            keypair=(pubkey, privkey)
+        )
+        self.failUnless(isinstance(node, dirnode.DirectoryNode))
+
+        dircap = uri.from_string(node.get_uri())
+        self.failUnless(isinstance(dircap, uri.DirectoryURI))
+
+        filecap = dircap.get_filenode_cap()
+        self.failUnless(isinstance(filecap, uri.WriteableSSKFileURI))
+
+        self.failUnlessReallyEqual(filecap.writekey, writekey)
+        self.failUnlessReallyEqual(filecap.fingerprint, fingerprint)
+
+        self.failUnlessReallyEqual(
+            # Despite being named "to_string", this actually returns bytes..
+            dircap.to_string(),
+            b'URI:DIR2:n4opqgewgcn4mddu4oiippaxru:ukpe4z6xdlujdpguoabergyih3bj7iaafukdqzwthy2ytdd5bs2a'
+        )

src/allmydata/web/unlinked.py

@@ -160,7 +160,7 @@ def POSTUnlinkedCreateDirectory(req, client):
     mt = None
     if file_format:
         mt = get_mutable_type(file_format)
-    d = client.create_dirnode(version=mt)
+    d = client.create_dirnode(version=mt, unique_keypair=get_keypair(req))
     redirect = get_arg(req, "redirect_to_result", "false")
     if boolean_of_arg(redirect):
         def _then_redir(res):
@@ -178,7 +178,7 @@ def POSTUnlinkedCreateDirectoryWithChildren(req, client):
     req.content.seek(0)
     kids_json = req.content.read()
     kids = convert_children_json(client.nodemaker, kids_json)
-    d = client.create_dirnode(initial_children=kids)
+    d = client.create_dirnode(initial_children=kids, unique_keypair=get_keypair(req))
     redirect = get_arg(req, "redirect_to_result", "false")
     if boolean_of_arg(redirect):
         def _then_redir(res):