mirror of
https://github.com/tahoe-lafs/tahoe-lafs.git
synced 2025-02-22 02:16:42 +00:00
Merge remote-tracking branch 'origin/master' into 3961.test-vectors
This commit is contained in:
Jean-Paul Calderone
commit
28e3188775
12
.github/workflows/ci.yml
vendored
12
.github/workflows/ci.yml
vendored
@ -86,8 +86,20 @@ jobs:
|
||||
run: python misc/build_helpers/show-tool-versions.py
|
||||
|
||||
- name: Run tox for corresponding Python version
|
||||
if: ${{ !contains(matrix.os, 'windows') }}
|
||||
run: python -m tox
|
||||
|
||||
# On Windows, a non-blocking pipe might respond (when emulating Unix-y
|
||||
# API) with ENOSPC to indicate buffer full. Trial doesn't handle this
|
||||
# well, so it breaks test runs. To attempt to solve this, we pipe the
|
||||
# output through passthrough.py that will hopefully be able to do the right
|
||||
# thing by using Windows APIs.
|
||||
- name: Run tox for corresponding Python version
|
||||
if: ${{ contains(matrix.os, 'windows') }}
|
||||
run: |
|
||||
pip install twisted pywin32
|
||||
python -m tox | python misc/windows-enospc/passthrough.py
|
||||
|
||||
- name: Upload eliot.log
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
|
||||
36
misc/windows-enospc/passthrough.py
Normal file
36
misc/windows-enospc/passthrough.py
Normal file
@ -0,0 +1,36 @@
|
||||
"""
|
||||
Writing to non-blocking pipe can result in ENOSPC when using Unix APIs on
|
||||
Windows. So, this program passes through data from stdin to stdout, using
|
||||
Windows APIs instead of Unix-y APIs.
|
||||
"""
|
||||
|
||||
from twisted.internet.stdio import StandardIO
|
||||
from twisted.internet import reactor
|
||||
from twisted.internet.protocol import Protocol
|
||||
from twisted.internet.interfaces import IHalfCloseableProtocol
|
||||
from twisted.internet.error import ReactorNotRunning
|
||||
from zope.interface import implementer
|
||||
|
||||
@implementer(IHalfCloseableProtocol)
|
||||
class Passthrough(Protocol):
|
||||
def readConnectionLost(self):
|
||||
self.transport.loseConnection()
|
||||
|
||||
def writeConnectionLost(self):
|
||||
try:
|
||||
reactor.stop()
|
||||
except ReactorNotRunning:
|
||||
pass
|
||||
|
||||
def dataReceived(self, data):
|
||||
self.transport.write(data)
|
||||
|
||||
def connectionLost(self, reason):
|
||||
try:
|
||||
reactor.stop()
|
||||
except ReactorNotRunning:
|
||||
pass
|
||||
|
||||
|
||||
std = StandardIO(Passthrough())
|
||||
reactor.run()
|
||||
0
newsfragments/3960.minor
Normal file
0
newsfragments/3960.minor
Normal file
1
newsfragments/3962.feature
Normal file
1
newsfragments/3962.feature
Normal file
@ -0,0 +1 @@
|
||||
Mutable objects can now be created with a pre-determined "signature key" using the ``tahoe put`` CLI or the HTTP API. This enables deterministic creation of mutable capabilities. This feature must be used with care to preserve the normal security and reliability properties.
|
||||
1
newsfragments/3966.bugfix
Normal file
1
newsfragments/3966.bugfix
Normal file
@ -0,0 +1 @@
|
||||
Fix incompatibility with newer versions of the transitive charset_normalizer dependency when using PyInstaller.
|
||||
7
setup.py
7
setup.py
@ -148,6 +148,13 @@ install_requires = [
|
||||
# for pid-file support
|
||||
"psutil",
|
||||
"filelock",
|
||||
|
||||
# treq needs requests, requests needs charset_normalizer,
|
||||
# charset_normalizer breaks PyInstaller
|
||||
# (https://github.com/Ousret/charset_normalizer/issues/253). So work around
|
||||
# this by using a lower version number. Once upstream issue is fixed, or
|
||||
# requests drops charset_normalizer, this can go away.
|
||||
"charset_normalizer < 3",
|
||||
]
|
||||
|
||||
setup_requires = [
|
||||
|
||||
@ -32,6 +32,7 @@ from allmydata.storage.server import StorageServer, FoolscapStorageServer
|
||||
from allmydata import storage_client
|
||||
from allmydata.immutable.upload import Uploader
|
||||
from allmydata.immutable.offloaded import Helper
|
||||
from allmydata.mutable.filenode import MutableFileNode
|
||||
from allmydata.introducer.client import IntroducerClient
|
||||
from allmydata.util import (
|
||||
hashutil, base32, pollmixin, log, idlib,
|
||||
@ -1086,9 +1087,40 @@ class _Client(node.Node, pollmixin.PollMixin):
|
||||
def create_immutable_dirnode(self, children, convergence=None):
|
||||
return self.nodemaker.create_immutable_directory(children, convergence)
|
||||
|
||||
def create_mutable_file(self, contents=None, version=None):
|
||||
def create_mutable_file(
|
||||
self,
|
||||
contents: bytes | None = None,
|
||||
version: int | None = None,
|
||||
*,
|
||||
unique_keypair: tuple[rsa.PublicKey, rsa.PrivateKey] | None = None,
|
||||
) -> MutableFileNode:
|
||||
"""
|
||||
Create *and upload* a new mutable object.
|
||||
|
||||
:param contents: If given, the initial contents for the new object.
|
||||
|
||||
:param version: If given, the mutable file format for the new object
|
||||
(otherwise a format will be chosen automatically).
|
||||
|
||||
:param unique_keypair: **Warning** This value independently determines
|
||||
the identity of the mutable object to create. There cannot be two
|
||||
different mutable objects that share a keypair. They will merge
|
||||
into one object (with undefined contents).
|
||||
|
||||
It is common to pass a None value (or not pass a valuye) for this
|
||||
parameter. In these cases, a new random keypair will be
|
||||
generated.
|
||||
|
||||
If non-None, the given public/private keypair will be used for the
|
||||
new object. The expected use-case is for implementing compliance
|
||||
tests.
|
||||
|
||||
:return: A Deferred which will fire with a representation of the new
|
||||
mutable object after it has been uploaded.
|
||||
"""
|
||||
return self.nodemaker.create_mutable_file(contents,
|
||||
version=version)
|
||||
version=version,
|
||||
keypair=unique_keypair)
|
||||
|
||||
def upload(self, uploadable, reactor=None):
|
||||
uploader = self.getServiceNamed("uploader")
|
||||
|
||||
@ -9,11 +9,13 @@ features of any objects that `cryptography` documents.
|
||||
|
||||
That is, the public and private keys are opaque objects; DO NOT depend
|
||||
on any of their methods.
|
||||
|
||||
Ported to Python 3.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing_extensions import TypeAlias
|
||||
from typing import Callable
|
||||
|
||||
from functools import partial
|
||||
|
||||
from cryptography.exceptions import InvalidSignature
|
||||
@ -25,6 +27,8 @@ from cryptography.hazmat.primitives.serialization import load_der_private_key, l
|
||||
|
||||
from allmydata.crypto.error import BadSignature
|
||||
|
||||
PublicKey: TypeAlias = rsa.RSAPublicKey
|
||||
PrivateKey: TypeAlias = rsa.RSAPrivateKey
|
||||
|
||||
# This is the value that was used by `pycryptopp`, and we must continue to use it for
|
||||
# both backwards compatibility and interoperability.
|
||||
@ -41,12 +45,12 @@ RSA_PADDING = padding.PSS(
|
||||
|
||||
|
||||
|
||||
def create_signing_keypair(key_size):
|
||||
def create_signing_keypair(key_size: int) -> tuple[PrivateKey, PublicKey]:
|
||||
"""
|
||||
Create a new RSA signing (private) keypair from scratch. Can be used with
|
||||
`sign_data` function.
|
||||
|
||||
:param int key_size: length of key in bits
|
||||
:param key_size: length of key in bits
|
||||
|
||||
:returns: 2-tuple of (private_key, public_key)
|
||||
"""
|
||||
@ -58,31 +62,42 @@ def create_signing_keypair(key_size):
|
||||
return priv_key, priv_key.public_key()
|
||||
|
||||
|
||||
def create_signing_keypair_from_string(private_key_der):
|
||||
def create_signing_keypair_from_string(private_key_der: bytes) -> tuple[PrivateKey, PublicKey]:
|
||||
"""
|
||||
Create an RSA signing (private) key from previously serialized
|
||||
private key bytes.
|
||||
|
||||
:param bytes private_key_der: blob as returned from `der_string_from_signing_keypair`
|
||||
:param private_key_der: blob as returned from `der_string_from_signing_keypair`
|
||||
|
||||
:returns: 2-tuple of (private_key, public_key)
|
||||
"""
|
||||
load = partial(
|
||||
_load = partial(
|
||||
load_der_private_key,
|
||||
private_key_der,
|
||||
password=None,
|
||||
backend=default_backend(),
|
||||
)
|
||||
|
||||
def load_with_validation() -> PrivateKey:
|
||||
k = _load()
|
||||
assert isinstance(k, PrivateKey)
|
||||
return k
|
||||
|
||||
def load_without_validation() -> PrivateKey:
|
||||
k = _load(unsafe_skip_rsa_key_validation=True)
|
||||
assert isinstance(k, PrivateKey)
|
||||
return k
|
||||
|
||||
# Load it once without the potentially expensive OpenSSL validation
|
||||
# checks. These have superlinear complexity. We *will* run them just
|
||||
# below - but first we'll apply our own constant-time checks.
|
||||
load: Callable[[], PrivateKey] = load_without_validation
|
||||
try:
|
||||
# Load it once without the potentially expensive OpenSSL validation
|
||||
# checks. These have superlinear complexity. We *will* run them just
|
||||
# below - but first we'll apply our own constant-time checks.
|
||||
unsafe_priv_key = load(unsafe_skip_rsa_key_validation=True)
|
||||
unsafe_priv_key = load()
|
||||
except TypeError:
|
||||
# cryptography<39 does not support this parameter, so just load the
|
||||
# key with validation...
|
||||
unsafe_priv_key = load()
|
||||
unsafe_priv_key = load_with_validation()
|
||||
# But avoid *reloading* it since that will run the expensive
|
||||
# validation *again*.
|
||||
load = lambda: unsafe_priv_key
|
||||
@ -102,7 +117,7 @@ def create_signing_keypair_from_string(private_key_der):
|
||||
return safe_priv_key, safe_priv_key.public_key()
|
||||
|
||||
|
||||
def der_string_from_signing_key(private_key):
|
||||
def der_string_from_signing_key(private_key: PrivateKey) -> bytes:
|
||||
"""
|
||||
Serializes a given RSA private key to a DER string
|
||||
|
||||
@ -112,14 +127,14 @@ def der_string_from_signing_key(private_key):
|
||||
:returns: bytes representing `private_key`
|
||||
"""
|
||||
_validate_private_key(private_key)
|
||||
return private_key.private_bytes(
|
||||
return private_key.private_bytes( # type: ignore[attr-defined]
|
||||
encoding=Encoding.DER,
|
||||
format=PrivateFormat.PKCS8,
|
||||
encryption_algorithm=NoEncryption(),
|
||||
)
|
||||
|
||||
|
||||
def der_string_from_verifying_key(public_key):
|
||||
def der_string_from_verifying_key(public_key: PublicKey) -> bytes:
|
||||
"""
|
||||
Serializes a given RSA public key to a DER string.
|
||||
|
||||
@ -135,7 +150,7 @@ def der_string_from_verifying_key(public_key):
|
||||
)
|
||||
|
||||
|
||||
def create_verifying_key_from_string(public_key_der):
|
||||
def create_verifying_key_from_string(public_key_der: bytes) -> PublicKey:
|
||||
"""
|
||||
Create an RSA verifying key from a previously serialized public key
|
||||
|
||||
@ -148,15 +163,16 @@ def create_verifying_key_from_string(public_key_der):
|
||||
public_key_der,
|
||||
backend=default_backend(),
|
||||
)
|
||||
assert isinstance(pub_key, PublicKey)
|
||||
return pub_key
|
||||
|
||||
|
||||
def sign_data(private_key, data):
|
||||
def sign_data(private_key: PrivateKey, data: bytes) -> bytes:
|
||||
"""
|
||||
:param private_key: the private part of a keypair returned from
|
||||
`create_signing_keypair_from_string` or `create_signing_keypair`
|
||||
|
||||
:param bytes data: the bytes to sign
|
||||
:param data: the bytes to sign
|
||||
|
||||
:returns: bytes which are a signature of the bytes given as `data`.
|
||||
"""
|
||||
@ -167,7 +183,7 @@ def sign_data(private_key, data):
|
||||
hashes.SHA256(),
|
||||
)
|
||||
|
||||
def verify_signature(public_key, alleged_signature, data):
|
||||
def verify_signature(public_key: PublicKey, alleged_signature: bytes, data: bytes) -> None:
|
||||
"""
|
||||
:param public_key: a verifying key, returned from `create_verifying_key_from_string` or `create_verifying_key_from_private_key`
|
||||
|
||||
@ -187,23 +203,23 @@ def verify_signature(public_key, alleged_signature, data):
|
||||
raise BadSignature()
|
||||
|
||||
|
||||
def _validate_public_key(public_key):
|
||||
def _validate_public_key(public_key: PublicKey) -> None:
|
||||
"""
|
||||
Internal helper. Checks that `public_key` is a valid cryptography
|
||||
object
|
||||
"""
|
||||
if not isinstance(public_key, rsa.RSAPublicKey):
|
||||
raise ValueError(
|
||||
"public_key must be an RSAPublicKey"
|
||||
f"public_key must be an RSAPublicKey not {type(public_key)}"
|
||||
)
|
||||
|
||||
|
||||
def _validate_private_key(private_key):
|
||||
def _validate_private_key(private_key: PrivateKey) -> None:
|
||||
"""
|
||||
Internal helper. Checks that `public_key` is a valid cryptography
|
||||
object
|
||||
"""
|
||||
if not isinstance(private_key, rsa.RSAPrivateKey):
|
||||
raise ValueError(
|
||||
"private_key must be an RSAPrivateKey"
|
||||
f"private_key must be an RSAPrivateKey not {type(private_key)}"
|
||||
)
|
||||
|
||||
@ -1,14 +1,7 @@
|
||||
"""
|
||||
Ported to Python 3.
|
||||
"""
|
||||
from __future__ import absolute_import
|
||||
from __future__ import division
|
||||
from __future__ import print_function
|
||||
from __future__ import unicode_literals
|
||||
|
||||
from future.utils import PY2
|
||||
if PY2:
|
||||
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
|
||||
from __future__ import annotations
|
||||
|
||||
MODE_CHECK = "MODE_CHECK" # query all peers
|
||||
MODE_ANYTHING = "MODE_ANYTHING" # one recoverable version
|
||||
@ -17,6 +10,9 @@ MODE_WRITE = "MODE_WRITE" # replace all shares, probably.. not for initial
|
||||
MODE_READ = "MODE_READ"
|
||||
MODE_REPAIR = "MODE_REPAIR" # query all peers, get the privkey
|
||||
|
||||
from allmydata.crypto import aes, rsa
|
||||
from allmydata.util import hashutil
|
||||
|
||||
class NotWriteableError(Exception):
|
||||
pass
|
||||
|
||||
@ -68,3 +64,33 @@ class CorruptShareError(BadShareError):
|
||||
|
||||
class UnknownVersionError(BadShareError):
|
||||
"""The share we received was of a version we don't recognize."""
|
||||
|
||||
|
||||
def encrypt_privkey(writekey: bytes, privkey: bytes) -> bytes:
|
||||
"""
|
||||
For SSK, encrypt a private ("signature") key using the writekey.
|
||||
"""
|
||||
encryptor = aes.create_encryptor(writekey)
|
||||
crypttext = aes.encrypt_data(encryptor, privkey)
|
||||
return crypttext
|
||||
|
||||
def decrypt_privkey(writekey: bytes, enc_privkey: bytes) -> rsa.PrivateKey:
|
||||
"""
|
||||
The inverse of ``encrypt_privkey``.
|
||||
"""
|
||||
decryptor = aes.create_decryptor(writekey)
|
||||
privkey = aes.decrypt_data(decryptor, enc_privkey)
|
||||
return privkey
|
||||
|
||||
def derive_mutable_keys(keypair: tuple[rsa.PublicKey, rsa.PrivateKey]) -> tuple[bytes, bytes, bytes]:
|
||||
"""
|
||||
Derive the SSK writekey, encrypted writekey, and fingerprint from the
|
||||
public/private ("verification" / "signature") keypair.
|
||||
"""
|
||||
pubkey, privkey = keypair
|
||||
pubkey_s = rsa.der_string_from_verifying_key(pubkey)
|
||||
privkey_s = rsa.der_string_from_signing_key(privkey)
|
||||
writekey = hashutil.ssk_writekey_hash(privkey_s)
|
||||
encprivkey = encrypt_privkey(writekey, privkey_s)
|
||||
fingerprint = hashutil.ssk_pubkey_fingerprint_hash(pubkey_s)
|
||||
return writekey, encprivkey, fingerprint
|
||||
|
||||
@ -1,14 +1,7 @@
|
||||
"""
|
||||
Ported to Python 3.
|
||||
"""
|
||||
from __future__ import absolute_import
|
||||
from __future__ import division
|
||||
from __future__ import print_function
|
||||
from __future__ import unicode_literals
|
||||
|
||||
from future.utils import PY2
|
||||
if PY2:
|
||||
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
|
||||
from __future__ import annotations
|
||||
|
||||
import random
|
||||
|
||||
@ -16,8 +9,6 @@ from zope.interface import implementer
|
||||
from twisted.internet import defer, reactor
|
||||
from foolscap.api import eventually
|
||||
|
||||
from allmydata.crypto import aes
|
||||
from allmydata.crypto import rsa
|
||||
from allmydata.interfaces import IMutableFileNode, ICheckable, ICheckResults, \
|
||||
NotEnoughSharesError, MDMF_VERSION, SDMF_VERSION, IMutableUploadable, \
|
||||
IMutableFileVersion, IWriteable
|
||||
@ -28,8 +19,14 @@ from allmydata.uri import WriteableSSKFileURI, ReadonlySSKFileURI, \
|
||||
from allmydata.monitor import Monitor
|
||||
from allmydata.mutable.publish import Publish, MutableData,\
|
||||
TransformingUploadable
|
||||
from allmydata.mutable.common import MODE_READ, MODE_WRITE, MODE_CHECK, UnrecoverableFileError, \
|
||||
UncoordinatedWriteError
|
||||
from allmydata.mutable.common import (
|
||||
MODE_READ,
|
||||
MODE_WRITE,
|
||||
MODE_CHECK,
|
||||
UnrecoverableFileError,
|
||||
UncoordinatedWriteError,
|
||||
derive_mutable_keys,
|
||||
)
|
||||
from allmydata.mutable.servermap import ServerMap, ServermapUpdater
|
||||
from allmydata.mutable.retrieve import Retrieve
|
||||
from allmydata.mutable.checker import MutableChecker, MutableCheckAndRepairer
|
||||
@ -139,13 +136,10 @@ class MutableFileNode(object):
|
||||
Deferred that fires (with the MutableFileNode instance you should
|
||||
use) when it completes.
|
||||
"""
|
||||
(pubkey, privkey) = keypair
|
||||
self._pubkey, self._privkey = pubkey, privkey
|
||||
pubkey_s = rsa.der_string_from_verifying_key(self._pubkey)
|
||||
privkey_s = rsa.der_string_from_signing_key(self._privkey)
|
||||
self._writekey = hashutil.ssk_writekey_hash(privkey_s)
|
||||
self._encprivkey = self._encrypt_privkey(self._writekey, privkey_s)
|
||||
self._fingerprint = hashutil.ssk_pubkey_fingerprint_hash(pubkey_s)
|
||||
self._pubkey, self._privkey = keypair
|
||||
self._writekey, self._encprivkey, self._fingerprint = derive_mutable_keys(
|
||||
keypair,
|
||||
)
|
||||
if version == MDMF_VERSION:
|
||||
self._uri = WriteableMDMFFileURI(self._writekey, self._fingerprint)
|
||||
self._protocol_version = version
|
||||
@ -171,16 +165,6 @@ class MutableFileNode(object):
|
||||
(contents, type(contents))
|
||||
return contents(self)
|
||||
|
||||
def _encrypt_privkey(self, writekey, privkey):
|
||||
encryptor = aes.create_encryptor(writekey)
|
||||
crypttext = aes.encrypt_data(encryptor, privkey)
|
||||
return crypttext
|
||||
|
||||
def _decrypt_privkey(self, enc_privkey):
|
||||
decryptor = aes.create_decryptor(self._writekey)
|
||||
privkey = aes.decrypt_data(decryptor, enc_privkey)
|
||||
return privkey
|
||||
|
||||
def _populate_pubkey(self, pubkey):
|
||||
self._pubkey = pubkey
|
||||
def _populate_required_shares(self, required_shares):
|
||||
|
||||
@ -1,15 +1,7 @@
|
||||
"""
|
||||
Ported to Python 3.
|
||||
"""
|
||||
from __future__ import absolute_import
|
||||
from __future__ import division
|
||||
from __future__ import print_function
|
||||
from __future__ import unicode_literals
|
||||
|
||||
from future.utils import PY2
|
||||
if PY2:
|
||||
# Don't import bytes and str, to prevent API leakage
|
||||
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, dict, list, object, range, max, min # noqa: F401
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
|
||||
@ -32,7 +24,7 @@ from allmydata import hashtree, codec
|
||||
from allmydata.storage.server import si_b2a
|
||||
|
||||
from allmydata.mutable.common import CorruptShareError, BadShareError, \
|
||||
UncoordinatedWriteError
|
||||
UncoordinatedWriteError, decrypt_privkey
|
||||
from allmydata.mutable.layout import MDMFSlotReadProxy
|
||||
|
||||
@implementer(IRetrieveStatus)
|
||||
@ -931,9 +923,10 @@ class Retrieve(object):
|
||||
|
||||
|
||||
def _try_to_validate_privkey(self, enc_privkey, reader, server):
|
||||
alleged_privkey_s = self._node._decrypt_privkey(enc_privkey)
|
||||
node_writekey = self._node.get_writekey()
|
||||
alleged_privkey_s = decrypt_privkey(node_writekey, enc_privkey)
|
||||
alleged_writekey = hashutil.ssk_writekey_hash(alleged_privkey_s)
|
||||
if alleged_writekey != self._node.get_writekey():
|
||||
if alleged_writekey != node_writekey:
|
||||
self.log("invalid privkey from %s shnum %d" %
|
||||
(reader, reader.shnum),
|
||||
level=log.WEIRD, umid="YIw4tA")
|
||||
|
||||
@ -1,16 +1,8 @@
|
||||
"""
|
||||
Ported to Python 3.
|
||||
"""
|
||||
from __future__ import print_function
|
||||
from __future__ import absolute_import
|
||||
from __future__ import division
|
||||
from __future__ import unicode_literals
|
||||
from __future__ import annotations
|
||||
|
||||
from future.utils import PY2
|
||||
if PY2:
|
||||
# Doesn't import str to prevent API leakage on Python 2
|
||||
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, max, min # noqa: F401
|
||||
from past.builtins import unicode
|
||||
from six import ensure_str
|
||||
|
||||
import sys, time, copy
|
||||
@ -29,7 +21,7 @@ from allmydata.storage.server import si_b2a
|
||||
from allmydata.interfaces import IServermapUpdaterStatus
|
||||
|
||||
from allmydata.mutable.common import MODE_CHECK, MODE_ANYTHING, MODE_WRITE, \
|
||||
MODE_READ, MODE_REPAIR, CorruptShareError
|
||||
MODE_READ, MODE_REPAIR, CorruptShareError, decrypt_privkey
|
||||
from allmydata.mutable.layout import SIGNED_PREFIX_LENGTH, MDMFSlotReadProxy
|
||||
|
||||
@implementer(IServermapUpdaterStatus)
|
||||
@ -203,8 +195,8 @@ class ServerMap(object):
|
||||
(seqnum, root_hash, IV, segsize, datalength, k, N, prefix,
|
||||
offsets_tuple) = verinfo
|
||||
print("[%s]: sh#%d seq%d-%s %d-of-%d len%d" %
|
||||
(unicode(server.get_name(), "utf-8"), shnum,
|
||||
seqnum, unicode(base32.b2a(root_hash)[:4], "utf-8"), k, N,
|
||||
(str(server.get_name(), "utf-8"), shnum,
|
||||
seqnum, str(base32.b2a(root_hash)[:4], "utf-8"), k, N,
|
||||
datalength), file=out)
|
||||
if self._problems:
|
||||
print("%d PROBLEMS" % len(self._problems), file=out)
|
||||
@ -276,7 +268,7 @@ class ServerMap(object):
|
||||
"""Take a versionid, return a string that describes it."""
|
||||
(seqnum, root_hash, IV, segsize, datalength, k, N, prefix,
|
||||
offsets_tuple) = verinfo
|
||||
return "seq%d-%s" % (seqnum, unicode(base32.b2a(root_hash)[:4], "utf-8"))
|
||||
return "seq%d-%s" % (seqnum, str(base32.b2a(root_hash)[:4], "utf-8"))
|
||||
|
||||
def summarize_versions(self):
|
||||
"""Return a string describing which versions we know about."""
|
||||
@ -824,7 +816,7 @@ class ServermapUpdater(object):
|
||||
|
||||
|
||||
def notify_server_corruption(self, server, shnum, reason):
|
||||
if isinstance(reason, unicode):
|
||||
if isinstance(reason, str):
|
||||
reason = reason.encode("utf-8")
|
||||
ss = server.get_storage_server()
|
||||
ss.advise_corrupt_share(
|
||||
@ -879,7 +871,7 @@ class ServermapUpdater(object):
|
||||
# ok, it's a valid verinfo. Add it to the list of validated
|
||||
# versions.
|
||||
self.log(" found valid version %d-%s from %s-sh%d: %d-%d/%d/%d"
|
||||
% (seqnum, unicode(base32.b2a(root_hash)[:4], "utf-8"),
|
||||
% (seqnum, str(base32.b2a(root_hash)[:4], "utf-8"),
|
||||
ensure_str(server.get_name()), shnum,
|
||||
k, n, segsize, datalen),
|
||||
parent=lp)
|
||||
@ -951,9 +943,10 @@ class ServermapUpdater(object):
|
||||
writekey stored in my node. If it is valid, then I set the
|
||||
privkey and encprivkey properties of the node.
|
||||
"""
|
||||
alleged_privkey_s = self._node._decrypt_privkey(enc_privkey)
|
||||
node_writekey = self._node.get_writekey()
|
||||
alleged_privkey_s = decrypt_privkey(node_writekey, enc_privkey)
|
||||
alleged_writekey = hashutil.ssk_writekey_hash(alleged_privkey_s)
|
||||
if alleged_writekey != self._node.get_writekey():
|
||||
if alleged_writekey != node_writekey:
|
||||
self.log("invalid privkey from %r shnum %d" %
|
||||
(server.get_name(), shnum),
|
||||
parent=lp, level=log.WEIRD, umid="aJVccw")
|
||||
|
||||
@ -1,17 +1,12 @@
|
||||
"""
|
||||
Ported to Python 3.
|
||||
Create file nodes of various types.
|
||||
"""
|
||||
from __future__ import absolute_import
|
||||
from __future__ import division
|
||||
from __future__ import print_function
|
||||
from __future__ import unicode_literals
|
||||
|
||||
from future.utils import PY2
|
||||
if PY2:
|
||||
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
|
||||
from __future__ import annotations
|
||||
|
||||
import weakref
|
||||
from zope.interface import implementer
|
||||
from twisted.internet.defer import succeed
|
||||
from allmydata.util.assertutil import precondition
|
||||
from allmydata.interfaces import INodeMaker
|
||||
from allmydata.immutable.literal import LiteralFileNode
|
||||
@ -22,6 +17,7 @@ from allmydata.mutable.publish import MutableData
|
||||
from allmydata.dirnode import DirectoryNode, pack_children
|
||||
from allmydata.unknown import UnknownNode
|
||||
from allmydata.blacklist import ProhibitedNode
|
||||
from allmydata.crypto.rsa import PublicKey, PrivateKey
|
||||
from allmydata import uri
|
||||
|
||||
|
||||
@ -126,12 +122,15 @@ class NodeMaker(object):
|
||||
return self._create_dirnode(filenode)
|
||||
return None
|
||||
|
||||
def create_mutable_file(self, contents=None, version=None):
|
||||
def create_mutable_file(self, contents=None, version=None, keypair: tuple[PublicKey, PrivateKey] | None = None):
|
||||
if version is None:
|
||||
version = self.mutable_file_default
|
||||
n = MutableFileNode(self.storage_broker, self.secret_holder,
|
||||
self.default_encoding_parameters, self.history)
|
||||
d = self.key_generator.generate()
|
||||
if keypair is None:
|
||||
d = self.key_generator.generate()
|
||||
else:
|
||||
d = succeed(keypair)
|
||||
d.addCallback(n.create_with_keys, contents, version=version)
|
||||
d.addCallback(lambda res: n)
|
||||
return d
|
||||
|
||||
@ -180,10 +180,22 @@ class GetOptions(FileStoreOptions):
|
||||
class PutOptions(FileStoreOptions):
|
||||
optFlags = [
|
||||
("mutable", "m", "Create a mutable file instead of an immutable one (like --format=SDMF)"),
|
||||
]
|
||||
]
|
||||
|
||||
optParameters = [
|
||||
("format", None, None, "Create a file with the given format: SDMF and MDMF for mutable, CHK (default) for immutable. (case-insensitive)"),
|
||||
]
|
||||
|
||||
("private-key-path", None, None,
|
||||
"***Warning*** "
|
||||
"It is possible to use this option to spoil the normal security properties of mutable objects. "
|
||||
"It is also possible to corrupt or destroy data with this option. "
|
||||
"Most users will not need this option and can ignore it. "
|
||||
"For mutables only, "
|
||||
"this gives a file containing a PEM-encoded 2048 bit RSA private key to use as the signature key for the mutable. "
|
||||
"The private key must be handled at least as strictly as the resulting capability string. "
|
||||
"A single private key must not be used for more than one mutable."
|
||||
),
|
||||
]
|
||||
|
||||
def parseArgs(self, arg1=None, arg2=None):
|
||||
# see Examples below
|
||||
|
||||
@ -1,23 +1,32 @@
|
||||
"""
|
||||
Ported to Python 3.
|
||||
Implement the ``tahoe put`` command.
|
||||
"""
|
||||
from __future__ import unicode_literals
|
||||
from __future__ import absolute_import
|
||||
from __future__ import division
|
||||
from __future__ import print_function
|
||||
|
||||
from future.utils import PY2
|
||||
if PY2:
|
||||
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
|
||||
from __future__ import annotations
|
||||
|
||||
from io import BytesIO
|
||||
from urllib.parse import quote as url_quote
|
||||
from base64 import urlsafe_b64encode
|
||||
|
||||
from cryptography.hazmat.primitives.serialization import load_pem_private_key
|
||||
|
||||
from twisted.python.filepath import FilePath
|
||||
|
||||
from allmydata.crypto.rsa import PrivateKey, der_string_from_signing_key
|
||||
from allmydata.scripts.common_http import do_http, format_http_success, format_http_error
|
||||
from allmydata.scripts.common import get_alias, DEFAULT_ALIAS, escape_path, \
|
||||
UnknownAliasError
|
||||
from allmydata.util.encodingutil import quote_output
|
||||
|
||||
def load_private_key(path: str) -> str:
|
||||
"""
|
||||
Load a private key from a file and return it in a format appropriate
|
||||
to include in the HTTP request.
|
||||
"""
|
||||
privkey = load_pem_private_key(FilePath(path).getContent(), password=None)
|
||||
assert isinstance(privkey, PrivateKey)
|
||||
derbytes = der_string_from_signing_key(privkey)
|
||||
return urlsafe_b64encode(derbytes).decode("ascii")
|
||||
|
||||
def put(options):
|
||||
"""
|
||||
@param verbosity: 0, 1, or 2, meaning quiet, verbose, or very verbose
|
||||
@ -29,6 +38,10 @@ def put(options):
|
||||
from_file = options.from_file
|
||||
to_file = options.to_file
|
||||
mutable = options['mutable']
|
||||
if options["private-key-path"] is None:
|
||||
private_key = None
|
||||
else:
|
||||
private_key = load_private_key(options["private-key-path"])
|
||||
format = options['format']
|
||||
if options['quiet']:
|
||||
verbosity = 0
|
||||
@ -79,6 +92,12 @@ def put(options):
|
||||
queryargs = []
|
||||
if mutable:
|
||||
queryargs.append("mutable=true")
|
||||
if private_key is not None:
|
||||
queryargs.append(f"private-key={private_key}")
|
||||
else:
|
||||
if private_key is not None:
|
||||
raise Exception("Can only supply a private key for mutables.")
|
||||
|
||||
if format:
|
||||
queryargs.append("format=%s" % format)
|
||||
if queryargs:
|
||||
@ -92,10 +111,7 @@ def put(options):
|
||||
if verbosity > 0:
|
||||
print("waiting for file data on stdin..", file=stderr)
|
||||
# We're uploading arbitrary files, so this had better be bytes:
|
||||
if PY2:
|
||||
stdinb = stdin
|
||||
else:
|
||||
stdinb = stdin.buffer
|
||||
stdinb = stdin.buffer
|
||||
data = stdinb.read()
|
||||
infileobj = BytesIO(data)
|
||||
|
||||
|
||||
@ -1,210 +0,0 @@
|
||||
"""
|
||||
This module is only necessary on Python 2. Once Python 2 code is dropped, it
|
||||
can be deleted.
|
||||
"""
|
||||
|
||||
from future.utils import PY3
|
||||
if PY3:
|
||||
raise RuntimeError("Just use subprocess.Popen")
|
||||
|
||||
# This is necessary to pacify flake8 on Python 3, while we're still supporting
|
||||
# Python 2.
|
||||
from past.builtins import unicode
|
||||
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
## Copyright (C) 2021 Valentin Lab
|
||||
##
|
||||
## Redistribution and use in source and binary forms, with or without
|
||||
## modification, are permitted provided that the following conditions
|
||||
## are met:
|
||||
##
|
||||
## 1. Redistributions of source code must retain the above copyright
|
||||
## notice, this list of conditions and the following disclaimer.
|
||||
##
|
||||
## 2. Redistributions in binary form must reproduce the above
|
||||
## copyright notice, this list of conditions and the following
|
||||
## disclaimer in the documentation and/or other materials provided
|
||||
## with the distribution.
|
||||
##
|
||||
## 3. Neither the name of the copyright holder nor the names of its
|
||||
## contributors may be used to endorse or promote products derived
|
||||
## from this software without specific prior written permission.
|
||||
##
|
||||
## THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
## "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
## LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
|
||||
## FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
|
||||
## COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
|
||||
## INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
|
||||
## (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
## SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
|
||||
## HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
## STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
## ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
|
||||
## OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
##
|
||||
|
||||
## issue: https://bugs.python.org/issue19264
|
||||
|
||||
# See allmydata/windows/fixups.py
|
||||
import sys
|
||||
assert sys.platform == "win32"
|
||||
|
||||
import os
|
||||
import ctypes
|
||||
import subprocess
|
||||
import _subprocess
|
||||
from ctypes import byref, windll, c_char_p, c_wchar_p, c_void_p, \
|
||||
Structure, sizeof, c_wchar, WinError
|
||||
from ctypes.wintypes import BYTE, WORD, LPWSTR, BOOL, DWORD, LPVOID, \
|
||||
HANDLE
|
||||
|
||||
|
||||
##
|
||||
## Types
|
||||
##
|
||||
|
||||
CREATE_UNICODE_ENVIRONMENT = 0x00000400
|
||||
LPCTSTR = c_char_p
|
||||
LPTSTR = c_wchar_p
|
||||
LPSECURITY_ATTRIBUTES = c_void_p
|
||||
LPBYTE = ctypes.POINTER(BYTE)
|
||||
|
||||
class STARTUPINFOW(Structure):
|
||||
_fields_ = [
|
||||
("cb", DWORD), ("lpReserved", LPWSTR),
|
||||
("lpDesktop", LPWSTR), ("lpTitle", LPWSTR),
|
||||
("dwX", DWORD), ("dwY", DWORD),
|
||||
("dwXSize", DWORD), ("dwYSize", DWORD),
|
||||
("dwXCountChars", DWORD), ("dwYCountChars", DWORD),
|
||||
("dwFillAtrribute", DWORD), ("dwFlags", DWORD),
|
||||
("wShowWindow", WORD), ("cbReserved2", WORD),
|
||||
("lpReserved2", LPBYTE), ("hStdInput", HANDLE),
|
||||
("hStdOutput", HANDLE), ("hStdError", HANDLE),
|
||||
]
|
||||
|
||||
LPSTARTUPINFOW = ctypes.POINTER(STARTUPINFOW)
|
||||
|
||||
|
||||
class PROCESS_INFORMATION(Structure):
|
||||
_fields_ = [
|
||||
("hProcess", HANDLE), ("hThread", HANDLE),
|
||||
("dwProcessId", DWORD), ("dwThreadId", DWORD),
|
||||
]
|
||||
|
||||
LPPROCESS_INFORMATION = ctypes.POINTER(PROCESS_INFORMATION)
|
||||
|
||||
|
||||
class DUMMY_HANDLE(ctypes.c_void_p):
|
||||
|
||||
def __init__(self, *a, **kw):
|
||||
super(DUMMY_HANDLE, self).__init__(*a, **kw)
|
||||
self.closed = False
|
||||
|
||||
def Close(self):
|
||||
if not self.closed:
|
||||
windll.kernel32.CloseHandle(self)
|
||||
self.closed = True
|
||||
|
||||
def __int__(self):
|
||||
return self.value
|
||||
|
||||
|
||||
CreateProcessW = windll.kernel32.CreateProcessW
|
||||
CreateProcessW.argtypes = [
|
||||
LPCTSTR, LPTSTR, LPSECURITY_ATTRIBUTES,
|
||||
LPSECURITY_ATTRIBUTES, BOOL, DWORD, LPVOID, LPCTSTR,
|
||||
LPSTARTUPINFOW, LPPROCESS_INFORMATION,
|
||||
]
|
||||
CreateProcessW.restype = BOOL
|
||||
|
||||
|
||||
##
|
||||
## Patched functions/classes
|
||||
##
|
||||
|
||||
def CreateProcess(executable, args, _p_attr, _t_attr,
|
||||
inherit_handles, creation_flags, env, cwd,
|
||||
startup_info):
|
||||
"""Create a process supporting unicode executable and args for win32
|
||||
|
||||
Python implementation of CreateProcess using CreateProcessW for Win32
|
||||
|
||||
"""
|
||||
|
||||
si = STARTUPINFOW(
|
||||
dwFlags=startup_info.dwFlags,
|
||||
wShowWindow=startup_info.wShowWindow,
|
||||
cb=sizeof(STARTUPINFOW),
|
||||
## XXXvlab: not sure of the casting here to ints.
|
||||
hStdInput=int(startup_info.hStdInput),
|
||||
hStdOutput=int(startup_info.hStdOutput),
|
||||
hStdError=int(startup_info.hStdError),
|
||||
)
|
||||
|
||||
wenv = None
|
||||
if env is not None:
|
||||
## LPCWSTR seems to be c_wchar_p, so let's say CWSTR is c_wchar
|
||||
env = (unicode("").join([
|
||||
unicode("%s=%s\0") % (k, v)
|
||||
for k, v in env.items()])) + unicode("\0")
|
||||
wenv = (c_wchar * len(env))()
|
||||
wenv.value = env
|
||||
|
||||
pi = PROCESS_INFORMATION()
|
||||
creation_flags |= CREATE_UNICODE_ENVIRONMENT
|
||||
|
||||
if CreateProcessW(executable, args, None, None,
|
||||
inherit_handles, creation_flags,
|
||||
wenv, cwd, byref(si), byref(pi)):
|
||||
return (DUMMY_HANDLE(pi.hProcess), DUMMY_HANDLE(pi.hThread),
|
||||
pi.dwProcessId, pi.dwThreadId)
|
||||
raise WinError()
|
||||
|
||||
|
||||
class Popen(subprocess.Popen):
|
||||
"""This superseeds Popen and corrects a bug in cPython 2.7 implem"""
|
||||
|
||||
def _execute_child(self, args, executable, preexec_fn, close_fds,
|
||||
cwd, env, universal_newlines,
|
||||
startupinfo, creationflags, shell, to_close,
|
||||
p2cread, p2cwrite,
|
||||
c2pread, c2pwrite,
|
||||
errread, errwrite):
|
||||
"""Code from part of _execute_child from Python 2.7 (9fbb65e)
|
||||
|
||||
There are only 2 little changes concerning the construction of
|
||||
the the final string in shell mode: we preempt the creation of
|
||||
the command string when shell is True, because original function
|
||||
will try to encode unicode args which we want to avoid to be able to
|
||||
sending it as-is to ``CreateProcess``.
|
||||
|
||||
"""
|
||||
if not isinstance(args, subprocess.types.StringTypes):
|
||||
args = subprocess.list2cmdline(args)
|
||||
|
||||
if startupinfo is None:
|
||||
startupinfo = subprocess.STARTUPINFO()
|
||||
if shell:
|
||||
startupinfo.dwFlags |= _subprocess.STARTF_USESHOWWINDOW
|
||||
startupinfo.wShowWindow = _subprocess.SW_HIDE
|
||||
comspec = os.environ.get("COMSPEC", unicode("cmd.exe"))
|
||||
args = unicode('{} /c "{}"').format(comspec, args)
|
||||
if (_subprocess.GetVersion() >= 0x80000000 or
|
||||
os.path.basename(comspec).lower() == "command.com"):
|
||||
w9xpopen = self._find_w9xpopen()
|
||||
args = unicode('"%s" %s') % (w9xpopen, args)
|
||||
creationflags |= _subprocess.CREATE_NEW_CONSOLE
|
||||
|
||||
cp = _subprocess.CreateProcess
|
||||
_subprocess.CreateProcess = CreateProcess
|
||||
try:
|
||||
super(Popen, self)._execute_child(
|
||||
args, executable,
|
||||
preexec_fn, close_fds, cwd, env, universal_newlines,
|
||||
startupinfo, creationflags, False, to_close, p2cread,
|
||||
p2cwrite, c2pread, c2pwrite, errread, errwrite,
|
||||
)
|
||||
finally:
|
||||
_subprocess.CreateProcess = cp
|
||||
@ -1,19 +1,18 @@
|
||||
"""
|
||||
Ported to Python 3.
|
||||
Tests for the ``tahoe put`` CLI tool.
|
||||
"""
|
||||
from __future__ import absolute_import
|
||||
from __future__ import division
|
||||
from __future__ import print_function
|
||||
from __future__ import unicode_literals
|
||||
|
||||
from future.utils import PY2
|
||||
if PY2:
|
||||
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Callable, Awaitable, TypeVar, Any
|
||||
import os.path
|
||||
from twisted.trial import unittest
|
||||
from twisted.python import usage
|
||||
from twisted.python.filepath import FilePath
|
||||
|
||||
from cryptography.hazmat.primitives.serialization import load_pem_private_key
|
||||
|
||||
from allmydata.crypto.rsa import PrivateKey
|
||||
from allmydata.uri import from_string
|
||||
from allmydata.util import fileutil
|
||||
from allmydata.scripts.common import get_aliases
|
||||
from allmydata.scripts import cli
|
||||
@ -22,6 +21,9 @@ from ..common_util import skip_if_cannot_represent_filename
|
||||
from allmydata.util.encodingutil import get_io_encoding
|
||||
from allmydata.util.fileutil import abspath_expanduser_unicode
|
||||
from .common import CLITestMixin
|
||||
from allmydata.mutable.common import derive_mutable_keys
|
||||
|
||||
T = TypeVar("T")
|
||||
|
||||
class Put(GridTestMixin, CLITestMixin, unittest.TestCase):
|
||||
|
||||
@ -215,6 +217,65 @@ class Put(GridTestMixin, CLITestMixin, unittest.TestCase):
|
||||
|
||||
return d
|
||||
|
||||
async def test_unlinked_mutable_specified_private_key(self) -> None:
|
||||
"""
|
||||
A new unlinked mutable can be created using a specified private
|
||||
key.
|
||||
"""
|
||||
self.basedir = "cli/Put/unlinked-mutable-with-key"
|
||||
await self._test_mutable_specified_key(
|
||||
lambda do_cli, pempath, datapath: do_cli(
|
||||
"put", "--mutable", "--private-key-path", pempath.path,
|
||||
stdin=datapath.getContent(),
|
||||
),
|
||||
)
|
||||
|
||||
async def test_linked_mutable_specified_private_key(self) -> None:
|
||||
"""
|
||||
A new linked mutable can be created using a specified private key.
|
||||
"""
|
||||
self.basedir = "cli/Put/linked-mutable-with-key"
|
||||
await self._test_mutable_specified_key(
|
||||
lambda do_cli, pempath, datapath: do_cli(
|
||||
"put", "--mutable", "--private-key-path", pempath.path, datapath.path,
|
||||
),
|
||||
)
|
||||
|
||||
async def _test_mutable_specified_key(
|
||||
self,
|
||||
run: Callable[[Any, FilePath, FilePath], Awaitable[tuple[int, bytes, bytes]]],
|
||||
) -> None:
|
||||
"""
|
||||
A helper for testing mutable creation.
|
||||
|
||||
:param run: A function to do the creation. It is called with
|
||||
``self.do_cli`` and the path to a private key PEM file and a data
|
||||
file. It returns whatever ``do_cli`` returns.
|
||||
"""
|
||||
self.set_up_grid(oneshare=True)
|
||||
|
||||
pempath = FilePath(__file__).parent().sibling("data").child("openssl-rsa-2048.txt")
|
||||
datapath = FilePath(self.basedir).child("data")
|
||||
datapath.setContent(b"Hello world" * 1024)
|
||||
|
||||
(rc, out, err) = await run(self.do_cli, pempath, datapath)
|
||||
self.assertEqual(rc, 0, (out, err))
|
||||
cap = from_string(out.strip())
|
||||
# The capability is derived from the key we specified.
|
||||
privkey = load_pem_private_key(pempath.getContent(), password=None)
|
||||
assert isinstance(privkey, PrivateKey)
|
||||
pubkey = privkey.public_key()
|
||||
writekey, _, fingerprint = derive_mutable_keys((pubkey, privkey))
|
||||
self.assertEqual(
|
||||
(writekey, fingerprint),
|
||||
(cap.writekey, cap.fingerprint),
|
||||
)
|
||||
# Also the capability we were given actually refers to the data we
|
||||
# uploaded.
|
||||
(rc, out, err) = await self.do_cli("get", out.strip())
|
||||
self.assertEqual(rc, 0, (out, err))
|
||||
self.assertEqual(out, datapath.getContent().decode("ascii"))
|
||||
|
||||
def test_mutable(self):
|
||||
# echo DATA1 | tahoe put --mutable - uploaded.txt
|
||||
# echo DATA2 | tahoe put - uploaded.txt # should modify-in-place
|
||||
|
||||
@ -1,14 +1,8 @@
|
||||
"""
|
||||
Ported to Python 3.
|
||||
Functionality related to a lot of the test suite.
|
||||
"""
|
||||
from __future__ import print_function
|
||||
from __future__ import absolute_import
|
||||
from __future__ import division
|
||||
from __future__ import unicode_literals
|
||||
from __future__ import annotations
|
||||
|
||||
from future.utils import PY2, native_str
|
||||
if PY2:
|
||||
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
|
||||
from past.builtins import chr as byteschr
|
||||
|
||||
__all__ = [
|
||||
@ -111,25 +105,15 @@ from allmydata.scripts.common import (
|
||||
|
||||
from ..crypto import (
|
||||
ed25519,
|
||||
rsa,
|
||||
)
|
||||
from .eliotutil import (
|
||||
EliotLoggedRunTest,
|
||||
)
|
||||
from .common_util import ShouldFailMixin # noqa: F401
|
||||
|
||||
if sys.platform == "win32" and PY2:
|
||||
# Python 2.7 doesn't have good options for launching a process with
|
||||
# non-ASCII in its command line. So use this alternative that does a
|
||||
# better job. However, only use it on Windows because it doesn't work
|
||||
# anywhere else.
|
||||
from ._win_subprocess import (
|
||||
Popen,
|
||||
)
|
||||
else:
|
||||
from subprocess import (
|
||||
Popen,
|
||||
)
|
||||
from subprocess import (
|
||||
Popen,
|
||||
PIPE,
|
||||
)
|
||||
|
||||
@ -298,7 +282,7 @@ class UseNode(object):
|
||||
plugin_config = attr.ib()
|
||||
storage_plugin = attr.ib()
|
||||
basedir = attr.ib(validator=attr.validators.instance_of(FilePath))
|
||||
introducer_furl = attr.ib(validator=attr.validators.instance_of(native_str),
|
||||
introducer_furl = attr.ib(validator=attr.validators.instance_of(str),
|
||||
converter=six.ensure_str)
|
||||
node_config = attr.ib(default=attr.Factory(dict))
|
||||
|
||||
@ -639,15 +623,28 @@ class FakeMutableFileNode(object): # type: ignore # incomplete implementation
|
||||
|
||||
MUTABLE_SIZELIMIT = 10000
|
||||
|
||||
def __init__(self, storage_broker, secret_holder,
|
||||
default_encoding_parameters, history, all_contents):
|
||||
_public_key: rsa.PublicKey | None
|
||||
_private_key: rsa.PrivateKey | None
|
||||
|
||||
def __init__(self,
|
||||
storage_broker,
|
||||
secret_holder,
|
||||
default_encoding_parameters,
|
||||
history,
|
||||
all_contents,
|
||||
keypair: tuple[rsa.PublicKey, rsa.PrivateKey] | None
|
||||
):
|
||||
self.all_contents = all_contents
|
||||
self.file_types = {} # storage index => MDMF_VERSION or SDMF_VERSION
|
||||
self.init_from_cap(make_mutable_file_cap())
|
||||
self.file_types: dict[bytes, int] = {} # storage index => MDMF_VERSION or SDMF_VERSION
|
||||
self.init_from_cap(make_mutable_file_cap(keypair))
|
||||
self._k = default_encoding_parameters['k']
|
||||
self._segsize = default_encoding_parameters['max_segment_size']
|
||||
def create(self, contents, key_generator=None, keysize=None,
|
||||
version=SDMF_VERSION):
|
||||
if keypair is None:
|
||||
self._public_key = self._private_key = None
|
||||
else:
|
||||
self._public_key, self._private_key = keypair
|
||||
|
||||
def create(self, contents, version=SDMF_VERSION):
|
||||
if version == MDMF_VERSION and \
|
||||
isinstance(self.my_uri, (uri.ReadonlySSKFileURI,
|
||||
uri.WriteableSSKFileURI)):
|
||||
@ -843,9 +840,28 @@ class FakeMutableFileNode(object): # type: ignore # incomplete implementation
|
||||
return defer.succeed(consumer)
|
||||
|
||||
|
||||
def make_mutable_file_cap():
|
||||
return uri.WriteableSSKFileURI(writekey=os.urandom(16),
|
||||
fingerprint=os.urandom(32))
|
||||
def make_mutable_file_cap(
|
||||
keypair: tuple[rsa.PublicKey, rsa.PrivateKey] | None = None,
|
||||
) -> uri.WriteableSSKFileURI:
|
||||
"""
|
||||
Create a local representation of a mutable object.
|
||||
|
||||
:param keypair: If None, a random keypair will be generated for the new
|
||||
object. Otherwise, this is the keypair for that object.
|
||||
"""
|
||||
if keypair is None:
|
||||
writekey = os.urandom(16)
|
||||
fingerprint = os.urandom(32)
|
||||
else:
|
||||
pubkey, privkey = keypair
|
||||
pubkey_s = rsa.der_string_from_verifying_key(pubkey)
|
||||
privkey_s = rsa.der_string_from_signing_key(privkey)
|
||||
writekey = hashutil.ssk_writekey_hash(privkey_s)
|
||||
fingerprint = hashutil.ssk_pubkey_fingerprint_hash(pubkey_s)
|
||||
|
||||
return uri.WriteableSSKFileURI(
|
||||
writekey=writekey, fingerprint=fingerprint,
|
||||
)
|
||||
|
||||
def make_mdmf_mutable_file_cap():
|
||||
return uri.WriteableMDMFFileURI(writekey=os.urandom(16),
|
||||
@ -875,7 +891,7 @@ def create_mutable_filenode(contents, mdmf=False, all_contents=None):
|
||||
encoding_params['max_segment_size'] = 128*1024
|
||||
|
||||
filenode = FakeMutableFileNode(None, None, encoding_params, None,
|
||||
all_contents)
|
||||
all_contents, None)
|
||||
filenode.init_from_cap(cap)
|
||||
if mdmf:
|
||||
filenode.create(MutableData(contents), version=MDMF_VERSION)
|
||||
|
||||
28
src/allmydata/test/data/openssl-rsa-2048.txt
Normal file
28
src/allmydata/test/data/openssl-rsa-2048.txt
Normal file
@ -0,0 +1,28 @@
|
||||
-----BEGIN PRIVATE KEY-----
|
||||
MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDF1MeXulDWFO05
|
||||
YXCh8aqNc1dS1ddJRzsti4BOWuDOepUc0oCaSIcC5aR7XJ+vhX7a02mTIwvLcuEH
|
||||
8sxx0BJU4jCDpRI6aAqaKJxwZx1e6AcVFJDl7vzymhvWhqHuKh0jTvwM2zONWTwV
|
||||
V8m2PbDdxu0Prwdx+Mt2sDT6xHEhJj5fI/GUDUEdkhLJF6DQSulFRqqd0qP7qcI9
|
||||
fSHZbM7MywfzqFUe8J1+tk4fBh2v7gNzN1INpzh2mDtLPAtxr4ZPtEb/0D0U4PsP
|
||||
CniOHP0U8sF3VY0+K5qoCQr92cLRJvT/vLpQGVNUTFdFrtbqDoFxUCyEH4FUqRDX
|
||||
2mVrPo2xAgMBAAECggEAA0Ev1y5/1NTPbgytBeIIH3d+v9hwKDbHecVoMwnOVeFJ
|
||||
BZpONrOToovhAc1NXH2wj4SvwYWfpJ1HR9piDAuLeKlnuUu4ffzfE0gQok4E+v4r
|
||||
2yg9ZcYBs/NOetAYVwbq960tiv/adFRr71E0WqbfS3fBx8q2L3Ujkkhd98PudUhQ
|
||||
izbrTvkT7q00OPCWGwgWepMlLEowUWwZehGI0MlbONg7SbRraZZmG586Iy0tpC3e
|
||||
AM7wC1/ORzFqcRgTIxXizQ5RHL7S0OQPLhbEJbuwPonNjze3p0EP4wNBELZTaVOd
|
||||
xeA22Py4Bh/d1q3aEgbwR7tLyA8YfEzshTaY6oV8AQKBgQD0uFo8pyWk0AWXfjzn
|
||||
jV4yYyPWy8pJA6YfAJAST8m7B/JeYgGlfHxTlNZiB40DsJq08tOZv3HAubgMpFIa
|
||||
reuDxPqo6/Quwdy4Syu+AFhY48KIuwuoegG/L+5qcQLE69r1w71ZV6wUvLmXYX2I
|
||||
Y6nYz+OdpD1JrMIr6Js60XURsQKBgQDO8yWl7ufIDKMbQpbs0PgUQsH4FtzGcP4J
|
||||
j/7/8GfhKYt6rPsrojPHUbAi1+25xBVOuhm0Zx2ku2t+xPIMJoS+15EcER1Z2iHZ
|
||||
Zci9UGpJpUxGcUhG7ETF1HZv0xKHcEOl9eIIOcAP9Vd9DqnGk85gy6ti6MHe/5Tn
|
||||
IMD36OQ8AQKBgQDwqE7NMM67KnslRNaeG47T3F0FQbm3XehCuqnz6BUJYcI+gQD/
|
||||
fdFB3K+LDcPmKgmqAtaGbxdtoPXXMM0xQXHHTrH15rxmMu1dK0dj/TDkkW7gSZko
|
||||
YHtRSdCbSnGfuBXG9GxD7QzkA8g7j3sE4oXIGoDLqRVAW61DwubMy+jlsQKBgGNB
|
||||
+Zepi1/Gt+BWQt8YpzPIhRIBnShMf3uEphCJdLlo3K4dE2btKBp8UpeTq0CDDJky
|
||||
5ytAndYp0jf+K/2p59dEuyOUDdjPp5aGnA446JGkB35tzPW/Uoj0C049FVEChl+u
|
||||
HBhH4peE285uXv2QXNbOOMh6zKmxOfDVI9iDyhwBAoGBAIXq2Ar0zDXXaL3ncEKo
|
||||
pXt9BZ8OpJo2pvB1t2VPePOwEQ0wdT+H62fKNY47NiF9+LyS541/ps5Qhv6AmiKJ
|
||||
Z7I0Vb6+sxQljYH/LNW+wc2T/pIAi/7sNcmnlBtZfoVwt99bk2CyoRALPLWHYCkh
|
||||
c7Tty2bZzDZy6aCX+FGRt5N/
|
||||
-----END PRIVATE KEY-----
|
||||
@ -30,6 +30,7 @@ from allmydata.mutable.publish import MutableData
|
||||
from ..test_download import PausingConsumer, PausingAndStoppingConsumer, \
|
||||
StoppingConsumer, ImmediatelyStoppingConsumer
|
||||
from .. import common_util as testutil
|
||||
from ...crypto.rsa import create_signing_keypair
|
||||
from .util import (
|
||||
FakeStorage,
|
||||
make_nodemaker_with_peers,
|
||||
@ -65,6 +66,16 @@ class Filenode(AsyncBrokenTestCase, testutil.ShouldFailMixin):
|
||||
d.addCallback(_created)
|
||||
return d
|
||||
|
||||
async def test_create_with_keypair(self):
|
||||
"""
|
||||
An SDMF can be created using a given keypair.
|
||||
"""
|
||||
(priv, pub) = create_signing_keypair(2048)
|
||||
node = await self.nodemaker.create_mutable_file(keypair=(pub, priv))
|
||||
self.assertThat(
|
||||
(node.get_privkey(), node.get_pubkey()),
|
||||
Equals((priv, pub)),
|
||||
)
|
||||
|
||||
def test_create_mdmf(self):
|
||||
d = self.nodemaker.create_mutable_file(version=MDMF_VERSION)
|
||||
|
||||
@ -1619,7 +1619,8 @@ class FakeMutableFile(object): # type: ignore # incomplete implementation
|
||||
return defer.succeed(None)
|
||||
|
||||
class FakeNodeMaker(NodeMaker):
|
||||
def create_mutable_file(self, contents=b"", keysize=None, version=None):
|
||||
def create_mutable_file(self, contents=b"", keysize=None, version=None, keypair=None):
|
||||
assert keypair is None, "FakeNodeMaker does not support externally supplied keypairs"
|
||||
return defer.succeed(FakeMutableFile(contents))
|
||||
|
||||
class FakeClient2(_Client): # type: ignore # tahoe-lafs/ticket/3573
|
||||
|
||||
@ -1,19 +1,14 @@
|
||||
"""
|
||||
Ported to Python 3.
|
||||
Tests for a bunch of web-related APIs.
|
||||
"""
|
||||
from __future__ import print_function
|
||||
from __future__ import absolute_import
|
||||
from __future__ import division
|
||||
from __future__ import unicode_literals
|
||||
from __future__ import annotations
|
||||
|
||||
from future.utils import PY2
|
||||
if PY2:
|
||||
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
|
||||
from six import ensure_binary
|
||||
|
||||
import os.path, re, time
|
||||
import treq
|
||||
from urllib.parse import quote as urlquote, unquote as urlunquote
|
||||
from base64 import urlsafe_b64encode
|
||||
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
@ -38,6 +33,7 @@ from allmydata.util import fileutil, base32, hashutil, jsonbytes as json
|
||||
from allmydata.util.consumer import download_to_data
|
||||
from allmydata.util.encodingutil import to_bytes
|
||||
from ...util.connection_status import ConnectionStatus
|
||||
from ...crypto.rsa import PublicKey, PrivateKey, create_signing_keypair, der_string_from_signing_key
|
||||
from ..common import (
|
||||
EMPTY_CLIENT_CONFIG,
|
||||
FakeCHKFileNode,
|
||||
@ -65,6 +61,7 @@ from allmydata.interfaces import (
|
||||
MustBeReadonlyError,
|
||||
)
|
||||
from allmydata.mutable import servermap, publish, retrieve
|
||||
from allmydata.mutable.common import derive_mutable_keys
|
||||
from .. import common_util as testutil
|
||||
from ..common_util import TimezoneMixin
|
||||
from ..common_web import (
|
||||
@ -93,6 +90,7 @@ class FakeNodeMaker(NodeMaker):
|
||||
'happy': 7,
|
||||
'max_segment_size':128*1024 # 1024=KiB
|
||||
}
|
||||
all_contents: dict[bytes, object]
|
||||
def _create_lit(self, cap):
|
||||
return FakeCHKFileNode(cap, self.all_contents)
|
||||
def _create_immutable(self, cap):
|
||||
@ -100,11 +98,19 @@ class FakeNodeMaker(NodeMaker):
|
||||
def _create_mutable(self, cap):
|
||||
return FakeMutableFileNode(None, None,
|
||||
self.encoding_params, None,
|
||||
self.all_contents).init_from_cap(cap)
|
||||
def create_mutable_file(self, contents=b"", keysize=None,
|
||||
version=SDMF_VERSION):
|
||||
self.all_contents, None).init_from_cap(cap)
|
||||
def create_mutable_file(self,
|
||||
contents=None,
|
||||
version=None,
|
||||
keypair: tuple[PublicKey, PrivateKey] | None=None,
|
||||
):
|
||||
if contents is None:
|
||||
contents = b""
|
||||
if version is None:
|
||||
version = SDMF_VERSION
|
||||
|
||||
n = FakeMutableFileNode(None, None, self.encoding_params, None,
|
||||
self.all_contents)
|
||||
self.all_contents, keypair)
|
||||
return n.create(contents, version=version)
|
||||
|
||||
class FakeUploader(service.Service):
|
||||
@ -2868,6 +2874,41 @@ class Web(WebMixin, WebErrorMixin, testutil.StallMixin, testutil.ReallyEqualMixi
|
||||
"Unknown format: foo",
|
||||
method="post", data=body, headers=headers)
|
||||
|
||||
async def test_POST_upload_keypair(self) -> None:
|
||||
"""
|
||||
A *POST* creating a new mutable object may include a *private-key*
|
||||
query argument giving a urlsafe-base64-encoded RSA private key to use
|
||||
as the "signature key". The given signature key is used, rather than
|
||||
a new one being generated.
|
||||
"""
|
||||
format = "sdmf"
|
||||
priv, pub = create_signing_keypair(2048)
|
||||
encoded_privkey = urlsafe_b64encode(der_string_from_signing_key(priv)).decode("ascii")
|
||||
filename = "predetermined-sdmf"
|
||||
expected_content = self.NEWFILE_CONTENTS * 100
|
||||
actual_cap = uri.from_string(await self.POST(
|
||||
self.public_url +
|
||||
f"/foo?t=upload&format={format}&private-key={encoded_privkey}",
|
||||
file=(filename, expected_content),
|
||||
))
|
||||
# Ideally we would inspect the private ("signature") and public
|
||||
# ("verification") keys but they are not made easily accessible here
|
||||
# (ostensibly because we have a FakeMutableFileNode instead of a real
|
||||
# one).
|
||||
#
|
||||
# So, instead, re-compute the writekey and fingerprint and compare
|
||||
# those against the capability string.
|
||||
expected_writekey, _, expected_fingerprint = derive_mutable_keys((pub, priv))
|
||||
self.assertEqual(
|
||||
(expected_writekey, expected_fingerprint),
|
||||
(actual_cap.writekey, actual_cap.fingerprint),
|
||||
)
|
||||
|
||||
# And the capability we got can be used to download the data we
|
||||
# uploaded.
|
||||
downloaded_content = await self.GET(f"/uri/{actual_cap.to_string().decode('ascii')}")
|
||||
self.assertEqual(expected_content, downloaded_content)
|
||||
|
||||
def test_POST_upload_format(self):
|
||||
def _check_upload(ign, format, uri_prefix, fn=None):
|
||||
filename = format + ".txt"
|
||||
|
||||
@ -202,6 +202,16 @@ class TahoeLAFSSiteTests(SyncTestCase):
|
||||
),
|
||||
)
|
||||
|
||||
def test_private_key_censoring(self):
|
||||
"""
|
||||
The log event for a request including a **private-key** query
|
||||
argument has the private key value censored.
|
||||
"""
|
||||
self._test_censoring(
|
||||
b"/uri?uri=URI:CHK:aaa:bbb&private-key=AAAAaaaabbbb==",
|
||||
b"/uri?uri=[CENSORED]&private-key=[CENSORED]",
|
||||
)
|
||||
|
||||
def test_uri_censoring(self):
|
||||
"""
|
||||
The log event for a request for **/uri/<CAP>** has the capability value
|
||||
|
||||
@ -1,26 +1,17 @@
|
||||
"""
|
||||
Ported to Python 3.
|
||||
"""
|
||||
from __future__ import division
|
||||
from __future__ import absolute_import
|
||||
from __future__ import print_function
|
||||
from __future__ import unicode_literals
|
||||
|
||||
from future.utils import PY2
|
||||
if PY2:
|
||||
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, max, min # noqa: F401
|
||||
from past.builtins import unicode as str # prevent leaking newbytes/newstr into code that can't handle it
|
||||
from __future__ import annotations
|
||||
|
||||
from six import ensure_str
|
||||
|
||||
try:
|
||||
from typing import Optional, Union, Tuple, Any
|
||||
except ImportError:
|
||||
pass
|
||||
from typing import Optional, Union, TypeVar, overload
|
||||
from typing_extensions import Literal
|
||||
|
||||
import time
|
||||
import json
|
||||
from functools import wraps
|
||||
from base64 import urlsafe_b64decode
|
||||
|
||||
from hyperlink import (
|
||||
DecodedURL,
|
||||
@ -94,7 +85,7 @@ from allmydata.util.encodingutil import (
|
||||
to_bytes,
|
||||
)
|
||||
from allmydata.util import abbreviate
|
||||
|
||||
from allmydata.crypto.rsa import PrivateKey, PublicKey, create_signing_keypair_from_string
|
||||
|
||||
class WebError(Exception):
|
||||
def __init__(self, text, code=http.BAD_REQUEST):
|
||||
@ -713,8 +704,15 @@ def url_for_string(req, url_string):
|
||||
)
|
||||
return url
|
||||
|
||||
T = TypeVar("T")
|
||||
|
||||
def get_arg(req, argname, default=None, multiple=False): # type: (IRequest, Union[bytes,str], Any, bool) -> Union[bytes,Tuple[bytes],Any]
|
||||
@overload
|
||||
def get_arg(req: IRequest, argname: str | bytes, default: T = None, *, multiple: Literal[False] = False) -> T | bytes: ...
|
||||
|
||||
@overload
|
||||
def get_arg(req: IRequest, argname: str | bytes, default: T = None, *, multiple: Literal[True]) -> T | tuple[bytes, ...]: ...
|
||||
|
||||
def get_arg(req: IRequest, argname: str | bytes, default: T = None, *, multiple: bool = False) -> None | T | bytes | tuple[bytes, ...]:
|
||||
"""Extract an argument from either the query args (req.args) or the form
|
||||
body fields (req.fields). If multiple=False, this returns a single value
|
||||
(or the default, which defaults to None), and the query args take
|
||||
@ -726,13 +724,14 @@ def get_arg(req, argname, default=None, multiple=False): # type: (IRequest, Uni
|
||||
:return: Either bytes or tuple of bytes.
|
||||
"""
|
||||
if isinstance(argname, str):
|
||||
argname = argname.encode("utf-8")
|
||||
if isinstance(default, str):
|
||||
default = default.encode("utf-8")
|
||||
argname_bytes = argname.encode("utf-8")
|
||||
else:
|
||||
argname_bytes = argname
|
||||
|
||||
results = []
|
||||
if argname in req.args:
|
||||
results.extend(req.args[argname])
|
||||
argname_unicode = str(argname, "utf-8")
|
||||
if argname_bytes in req.args:
|
||||
results.extend(req.args[argname_bytes])
|
||||
argname_unicode = str(argname_bytes, "utf-8")
|
||||
if req.fields and argname_unicode in req.fields:
|
||||
value = req.fields[argname_unicode].value
|
||||
if isinstance(value, str):
|
||||
@ -742,6 +741,9 @@ def get_arg(req, argname, default=None, multiple=False): # type: (IRequest, Uni
|
||||
return tuple(results)
|
||||
if results:
|
||||
return results[0]
|
||||
|
||||
if isinstance(default, str):
|
||||
return default.encode("utf-8")
|
||||
return default
|
||||
|
||||
|
||||
@ -833,3 +835,14 @@ def abbreviate_time(data):
|
||||
if s >= 0.001:
|
||||
return u"%.1fms" % (1000*s)
|
||||
return u"%.0fus" % (1000000*s)
|
||||
|
||||
def get_keypair(request: IRequest) -> tuple[PublicKey, PrivateKey] | None:
|
||||
"""
|
||||
Load a keypair from a urlsafe-base64-encoded RSA private key in the
|
||||
**private-key** argument of the given request, if there is one.
|
||||
"""
|
||||
privkey_der = get_arg(request, "private-key", default=None, multiple=False)
|
||||
if privkey_der is None:
|
||||
return None
|
||||
privkey, pubkey = create_signing_keypair_from_string(urlsafe_b64decode(privkey_der))
|
||||
return pubkey, privkey
|
||||
|
||||
@ -1,23 +1,12 @@
|
||||
"""
|
||||
Ported to Python 3.
|
||||
"""
|
||||
from __future__ import absolute_import
|
||||
from __future__ import division
|
||||
from __future__ import print_function
|
||||
from __future__ import unicode_literals
|
||||
|
||||
from future.utils import PY2
|
||||
if PY2:
|
||||
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, max, min # noqa: F401
|
||||
# Use native unicode() as str() to prevent leaking futurebytes in ways that
|
||||
# break string formattin.
|
||||
from past.builtins import unicode as str
|
||||
from past.builtins import long
|
||||
from __future__ import annotations
|
||||
|
||||
from twisted.web import http, static
|
||||
from twisted.internet import defer
|
||||
from twisted.web.resource import (
|
||||
Resource, # note: Resource is an old-style class
|
||||
Resource,
|
||||
ErrorPage,
|
||||
)
|
||||
|
||||
@ -34,6 +23,7 @@ from allmydata.blacklist import (
|
||||
)
|
||||
|
||||
from allmydata.web.common import (
|
||||
get_keypair,
|
||||
boolean_of_arg,
|
||||
exception_to_child,
|
||||
get_arg,
|
||||
@ -56,7 +46,6 @@ from allmydata.web.check_results import (
|
||||
from allmydata.web.info import MoreInfo
|
||||
from allmydata.util import jsonbytes as json
|
||||
|
||||
|
||||
class ReplaceMeMixin(object):
|
||||
def replace_me_with_a_child(self, req, client, replace):
|
||||
# a new file is being uploaded in our place.
|
||||
@ -64,7 +53,8 @@ class ReplaceMeMixin(object):
|
||||
mutable_type = get_mutable_type(file_format)
|
||||
if mutable_type is not None:
|
||||
data = MutableFileHandle(req.content)
|
||||
d = client.create_mutable_file(data, version=mutable_type)
|
||||
keypair = get_keypair(req)
|
||||
d = client.create_mutable_file(data, version=mutable_type, unique_keypair=keypair)
|
||||
def _uploaded(newnode):
|
||||
d2 = self.parentnode.set_node(self.name, newnode,
|
||||
overwrite=replace)
|
||||
@ -106,7 +96,8 @@ class ReplaceMeMixin(object):
|
||||
if file_format in ("SDMF", "MDMF"):
|
||||
mutable_type = get_mutable_type(file_format)
|
||||
uploadable = MutableFileHandle(contents.file)
|
||||
d = client.create_mutable_file(uploadable, version=mutable_type)
|
||||
keypair = get_keypair(req)
|
||||
d = client.create_mutable_file(uploadable, version=mutable_type, unique_keypair=keypair)
|
||||
def _uploaded(newnode):
|
||||
d2 = self.parentnode.set_node(self.name, newnode,
|
||||
overwrite=replace)
|
||||
@ -395,7 +386,7 @@ class FileDownloader(Resource, object):
|
||||
# list of (first,last) inclusive range tuples.
|
||||
|
||||
filesize = self.filenode.get_size()
|
||||
assert isinstance(filesize, (int,long)), filesize
|
||||
assert isinstance(filesize, int), filesize
|
||||
|
||||
try:
|
||||
# byte-ranges-specifier
|
||||
@ -408,19 +399,19 @@ class FileDownloader(Resource, object):
|
||||
|
||||
if first == '':
|
||||
# suffix-byte-range-spec
|
||||
first = filesize - long(last)
|
||||
first = filesize - int(last)
|
||||
last = filesize - 1
|
||||
else:
|
||||
# byte-range-spec
|
||||
|
||||
# first-byte-pos
|
||||
first = long(first)
|
||||
first = int(first)
|
||||
|
||||
# last-byte-pos
|
||||
if last == '':
|
||||
last = filesize - 1
|
||||
else:
|
||||
last = long(last)
|
||||
last = int(last)
|
||||
|
||||
if last < first:
|
||||
raise ValueError
|
||||
@ -456,7 +447,7 @@ class FileDownloader(Resource, object):
|
||||
b'attachment; filename="%s"' % self.filename)
|
||||
|
||||
filesize = self.filenode.get_size()
|
||||
assert isinstance(filesize, (int,long)), filesize
|
||||
assert isinstance(filesize, int), filesize
|
||||
first, size = 0, None
|
||||
contentsize = filesize
|
||||
req.setHeader("accept-ranges", "bytes")
|
||||
|
||||
@ -1,14 +1,7 @@
|
||||
"""
|
||||
Ported to Python 3.
|
||||
"""
|
||||
from __future__ import absolute_import
|
||||
from __future__ import division
|
||||
from __future__ import print_function
|
||||
from __future__ import unicode_literals
|
||||
|
||||
from future.utils import PY2
|
||||
if PY2:
|
||||
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
|
||||
from __future__ import annotations
|
||||
|
||||
from urllib.parse import quote as urlquote
|
||||
|
||||
@ -25,6 +18,7 @@ from twisted.web.template import (
|
||||
from allmydata.immutable.upload import FileHandle
|
||||
from allmydata.mutable.publish import MutableFileHandle
|
||||
from allmydata.web.common import (
|
||||
get_keypair,
|
||||
get_arg,
|
||||
boolean_of_arg,
|
||||
convert_children_json,
|
||||
@ -48,7 +42,8 @@ def PUTUnlinkedSSK(req, client, version):
|
||||
# SDMF: files are small, and we can only upload data
|
||||
req.content.seek(0)
|
||||
data = MutableFileHandle(req.content)
|
||||
d = client.create_mutable_file(data, version=version)
|
||||
keypair = get_keypair(req)
|
||||
d = client.create_mutable_file(data, version=version, unique_keypair=keypair)
|
||||
d.addCallback(lambda n: n.get_uri())
|
||||
return d
|
||||
|
||||
|
||||
@ -1,18 +1,12 @@
|
||||
"""
|
||||
Ported to Python 3.
|
||||
General web server-related utilities.
|
||||
"""
|
||||
from __future__ import absolute_import
|
||||
from __future__ import division
|
||||
from __future__ import print_function
|
||||
from __future__ import unicode_literals
|
||||
|
||||
from future.utils import PY2
|
||||
if PY2:
|
||||
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
|
||||
from __future__ import annotations
|
||||
|
||||
from six import ensure_str
|
||||
|
||||
import re, time, tempfile
|
||||
from urllib.parse import parse_qsl, urlencode
|
||||
|
||||
from cgi import (
|
||||
FieldStorage,
|
||||
@ -45,40 +39,37 @@ from .web.storage_plugins import (
|
||||
)
|
||||
|
||||
|
||||
if PY2:
|
||||
FileUploadFieldStorage = FieldStorage
|
||||
else:
|
||||
class FileUploadFieldStorage(FieldStorage):
|
||||
"""
|
||||
Do terrible things to ensure files are still bytes.
|
||||
class FileUploadFieldStorage(FieldStorage):
|
||||
"""
|
||||
Do terrible things to ensure files are still bytes.
|
||||
|
||||
On Python 2, uploaded files were always bytes. On Python 3, there's a
|
||||
heuristic: if the filename is set on a field, it's assumed to be a file
|
||||
upload and therefore bytes. If no filename is set, it's Unicode.
|
||||
On Python 2, uploaded files were always bytes. On Python 3, there's a
|
||||
heuristic: if the filename is set on a field, it's assumed to be a file
|
||||
upload and therefore bytes. If no filename is set, it's Unicode.
|
||||
|
||||
Unfortunately, we always want it to be bytes, and Tahoe-LAFS also
|
||||
enables setting the filename not via the MIME filename, but via a
|
||||
separate field called "name".
|
||||
Unfortunately, we always want it to be bytes, and Tahoe-LAFS also
|
||||
enables setting the filename not via the MIME filename, but via a
|
||||
separate field called "name".
|
||||
|
||||
Thus we need to do this ridiculous workaround. Mypy doesn't like it
|
||||
either, thus the ``# type: ignore`` below.
|
||||
Thus we need to do this ridiculous workaround. Mypy doesn't like it
|
||||
either, thus the ``# type: ignore`` below.
|
||||
|
||||
Source for idea:
|
||||
https://mail.python.org/pipermail/python-dev/2017-February/147402.html
|
||||
"""
|
||||
@property # type: ignore
|
||||
def filename(self):
|
||||
if self.name == "file" and not self._mime_filename:
|
||||
# We use the file field to upload files, see directory.py's
|
||||
# _POST_upload. Lack of _mime_filename means we need to trick
|
||||
# FieldStorage into thinking there is a filename so it'll
|
||||
# return bytes.
|
||||
return "unknown-filename"
|
||||
return self._mime_filename
|
||||
Source for idea:
|
||||
https://mail.python.org/pipermail/python-dev/2017-February/147402.html
|
||||
"""
|
||||
@property # type: ignore
|
||||
def filename(self):
|
||||
if self.name == "file" and not self._mime_filename:
|
||||
# We use the file field to upload files, see directory.py's
|
||||
# _POST_upload. Lack of _mime_filename means we need to trick
|
||||
# FieldStorage into thinking there is a filename so it'll
|
||||
# return bytes.
|
||||
return "unknown-filename"
|
||||
return self._mime_filename
|
||||
|
||||
@filename.setter
|
||||
def filename(self, value):
|
||||
self._mime_filename = value
|
||||
@filename.setter
|
||||
def filename(self, value):
|
||||
self._mime_filename = value
|
||||
|
||||
|
||||
class TahoeLAFSRequest(Request, object):
|
||||
@ -180,12 +171,7 @@ def _logFormatter(logDateTime, request):
|
||||
queryargs = b""
|
||||
else:
|
||||
path, queryargs = x
|
||||
# there is a form handler which redirects POST /uri?uri=FOO into
|
||||
# GET /uri/FOO so folks can paste in non-HTTP-prefixed uris. Make
|
||||
# sure we censor these too.
|
||||
if queryargs.startswith(b"uri="):
|
||||
queryargs = b"uri=[CENSORED]"
|
||||
queryargs = b"?" + queryargs
|
||||
queryargs = b"?" + censor(queryargs)
|
||||
if path.startswith(b"/uri/"):
|
||||
path = b"/uri/[CENSORED]"
|
||||
elif path.startswith(b"/file/"):
|
||||
@ -207,6 +193,30 @@ def _logFormatter(logDateTime, request):
|
||||
)
|
||||
|
||||
|
||||
def censor(queryargs: bytes) -> bytes:
|
||||
"""
|
||||
Replace potentially sensitive values in query arguments with a
|
||||
constant string.
|
||||
"""
|
||||
args = parse_qsl(queryargs.decode("ascii"), keep_blank_values=True, encoding="utf8")
|
||||
result = []
|
||||
for k, v in args:
|
||||
if k == "uri":
|
||||
# there is a form handler which redirects POST /uri?uri=FOO into
|
||||
# GET /uri/FOO so folks can paste in non-HTTP-prefixed uris. Make
|
||||
# sure we censor these.
|
||||
v = "[CENSORED]"
|
||||
elif k == "private-key":
|
||||
# Likewise, sometimes a private key is supplied with mutable
|
||||
# creation.
|
||||
v = "[CENSORED]"
|
||||
|
||||
result.append((k, v))
|
||||
|
||||
# Customize safe to try to leave our markers intact.
|
||||
return urlencode(result, safe="[]").encode("ascii")
|
||||
|
||||
|
||||
class TahoeLAFSSite(Site, object):
|
||||
"""
|
||||
The HTTP protocol factory used by Tahoe-LAFS.
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user