From 456df65a07a0c48f3a056519282cc96b5e4e2f25 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Fri, 5 Nov 2021 14:16:43 -0400 Subject: [PATCH] Add v2 of the mutable container schema It uses hashed lease secrets, like v2 of the immutable container schema. --- src/allmydata/storage/mutable_schema.py | 225 ++++++++++++++++++++---- 1 file changed, 187 insertions(+), 38 deletions(-) diff --git a/src/allmydata/storage/mutable_schema.py b/src/allmydata/storage/mutable_schema.py index 25f24ea1f..9496fe571 100644 --- a/src/allmydata/storage/mutable_schema.py +++ b/src/allmydata/storage/mutable_schema.py @@ -13,23 +13,193 @@ if PY2: import struct +try: + from typing import Union +except ImportError: + pass + +import attr + +from nacl.hash import blake2b +from nacl.encoding import RawEncoder + +from ..util.hashutil import ( + tagged_hash, +) from .lease import ( LeaseInfo, + HashedLeaseInfo, ) +def _magic(version): + # type: (int) -> bytes + """ + Compute a "magic" header string for a container of the given version. + + :param version: The version number of the container. + """ + # Make it easy for people to recognize + human_readable = u"Tahoe mutable container v{:d}\n".format(version).encode("ascii") + # But also keep the chance of accidental collision low + if version == 1: + # It's unclear where this byte sequence came from. It may have just + # been random. In any case, preserve it since it is the magic marker + # in all v1 share files. + random_bytes = b"\x75\x09\x44\x03\x8e" + else: + # For future versions, use a reproducable scheme. + random_bytes = tagged_hash( + b"allmydata_mutable_container_header", + human_readable, + truncate_to=5, + ) + magic = human_readable + random_bytes + assert len(magic) == 32 + if version > 1: + # The chance of collision is pretty low but let's just be sure about + # it. + assert magic != _magic(version - 1) + + return magic + +def _header(magic, extra_lease_offset, nodeid, write_enabler): + # type: (bytes, int, bytes, bytes) -> bytes + """ + Construct a container header. + + :param nodeid: A unique identifier for the node holding this + container. + + :param write_enabler: A secret shared with the client used to + authorize changes to the contents of this container. + """ + fixed_header = struct.pack( + ">32s20s32sQQ", + magic, + nodeid, + write_enabler, + # data length, initially the container is empty + 0, + extra_lease_offset, + ) + blank_leases = b"\x00" * LeaseInfo().mutable_size() * 4 + extra_lease_count = struct.pack(">L", 0) + + return b"".join([ + fixed_header, + # share data will go in between the next two items eventually but + # for now there is none. + blank_leases, + extra_lease_count, + ]) + + +class _V2(object): + """ + Implement encoding and decoding for v2 of the mutable container. + """ + version = 2 + _MAGIC = _magic(version) + + _HEADER_FORMAT = ">32s20s32sQQ" + + # This size excludes leases + _HEADER_SIZE = struct.calcsize(_HEADER_FORMAT) + + _EXTRA_LEASE_OFFSET = _HEADER_SIZE + 4 * LeaseInfo().mutable_size() + + @classmethod + def _hash_secret(cls, secret): + # type: (bytes) -> bytes + """ + Hash a lease secret for storage. + """ + return blake2b(secret, digest_size=32, encoder=RawEncoder()) + + @classmethod + def _hash_lease_info(cls, lease_info): + # type: (LeaseInfo) -> HashedLeaseInfo + """ + Hash the cleartext lease info secrets into a ``HashedLeaseInfo``. + """ + if not isinstance(lease_info, LeaseInfo): + # Provide a little safety against misuse, especially an attempt to + # re-hash an already-hashed lease info which is represented as a + # different type. + raise TypeError( + "Can only hash LeaseInfo, not {!r}".format(lease_info), + ) + + # Hash the cleartext secrets in the lease info and wrap the result in + # a new type. + return HashedLeaseInfo( + attr.assoc( + lease_info, + renew_secret=cls._hash_secret(lease_info.renew_secret), + cancel_secret=cls._hash_secret(lease_info.cancel_secret), + ), + cls._hash_secret, + ) + + @classmethod + def magic_matches(cls, candidate_magic): + # type: (bytes) -> bool + """ + Return ``True`` if a candidate string matches the expected magic string + from a mutable container header, ``False`` otherwise. + """ + return candidate_magic[:len(cls._MAGIC)] == cls._MAGIC + + @classmethod + def header(cls, nodeid, write_enabler): + return _header(cls._MAGIC, cls._EXTRA_LEASE_OFFSET, nodeid, write_enabler) + + @classmethod + def serialize_lease(cls, lease): + # type: (Union[LeaseInfo, HashedLeaseInfo]) -> bytes + """ + Serialize a lease to be written to a v2 container. + + :param lease: the lease to serialize + + :return: the serialized bytes + """ + if isinstance(lease, LeaseInfo): + # v2 of the mutable schema stores lease secrets hashed. If we're + # given a LeaseInfo then it holds plaintext secrets. Hash them + # before trying to serialize. + lease = cls._hash_lease_info(lease) + if isinstance(lease, HashedLeaseInfo): + return lease.to_mutable_data() + raise ValueError( + "MutableShareFile v2 schema cannot represent lease {!r}".format( + lease, + ), + ) + + @classmethod + def unserialize_lease(cls, data): + # type: (bytes) -> HashedLeaseInfo + """ + Unserialize some bytes from a v2 container. + + :param data: the bytes from the container + + :return: the ``HashedLeaseInfo`` the bytes represent + """ + # In v2 of the immutable schema lease secrets are stored hashed. Wrap + # a LeaseInfo in a HashedLeaseInfo so it can supply the correct + # interpretation for those values. + lease = LeaseInfo.from_mutable_data(data) + return HashedLeaseInfo(lease, cls._hash_secret) + + class _V1(object): """ Implement encoding and decoding for v1 of the mutable container. """ version = 1 - - _MAGIC = ( - # Make it easy for people to recognize - b"Tahoe mutable container v1\n" - # But also keep the chance of accidental collision low - b"\x75\x09\x44\x03\x8e" - ) - assert len(_MAGIC) == 32 + _MAGIC = _magic(version) _HEADER_FORMAT = ">32s20s32sQQ" @@ -49,35 +219,8 @@ class _V1(object): @classmethod def header(cls, nodeid, write_enabler): - # type: (bytes, bytes) -> bytes - """ - Construct a container header. + return _header(cls._MAGIC, cls._EXTRA_LEASE_OFFSET, nodeid, write_enabler) - :param nodeid: A unique identifier for the node holding this - container. - - :param write_enabler: A secret shared with the client used to - authorize changes to the contents of this container. - """ - fixed_header = struct.pack( - ">32s20s32sQQ", - cls._MAGIC, - nodeid, - write_enabler, - # data length, initially the container is empty - 0, - cls._EXTRA_LEASE_OFFSET, - ) - blank_leases = b"\x00" * LeaseInfo().mutable_size() * 4 - extra_lease_count = struct.pack(">L", 0) - - return b"".join([ - fixed_header, - # share data will go in between the next two items eventually but - # for now there is none. - blank_leases, - extra_lease_count, - ]) @classmethod def serialize_lease(cls, lease_info): @@ -89,7 +232,13 @@ class _V1(object): :return: the serialized bytes """ - return lease_info.to_mutable_data() + if isinstance(lease, LeaseInfo): + return lease_info.to_mutable_data() + raise ValueError( + "MutableShareFile v1 schema only supports LeaseInfo, not {!r}".format( + lease, + ), + ) @classmethod def unserialize_lease(cls, data): @@ -104,7 +253,7 @@ class _V1(object): return LeaseInfo.from_mutable_data(data) -ALL_SCHEMAS = {_V1} +ALL_SCHEMAS = {_V2, _V1} ALL_SCHEMA_VERSIONS = {schema.version for schema in ALL_SCHEMAS} # type: ignore NEWEST_SCHEMA_VERSION = max(ALL_SCHEMAS, key=lambda schema: schema.version) # type: ignore