mirror of
https://github.com/tahoe-lafs/tahoe-lafs.git
synced 2024-12-21 13:57:51 +00:00
Move some general utility functions into the util module
This commit is contained in:
Jean-Paul Calderone
parent
43388ee711
commit
8a42720301
@ -4,9 +4,7 @@ Verify certain results against test vectors with well-known results.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TypeVar, Iterator, Awaitable, Callable
|
||||
|
||||
from tempfile import NamedTemporaryFile
|
||||
from typing import AsyncGenerator
|
||||
from hashlib import sha256
|
||||
from itertools import product
|
||||
from yaml import safe_dump
|
||||
@ -15,13 +13,16 @@ from pytest import mark
|
||||
from pytest_twisted import ensureDeferred
|
||||
|
||||
from . import vectors
|
||||
from .util import cli, await_client_ready
|
||||
from allmydata.client import read_config
|
||||
from allmydata.util import base32
|
||||
from .util import reconfigure, upload, asyncfoldr, insert, TahoeProcess
|
||||
|
||||
def digest(bs: bytes) -> str:
|
||||
def digest(bs: bytes) -> bytes:
|
||||
return sha256(bs).digest()
|
||||
|
||||
|
||||
def hexdigest(bs: bytes) -> str:
|
||||
return sha256(bs).hexdigest()
|
||||
|
||||
|
||||
CONVERGENCE_SECRETS = [
|
||||
b"aaaaaaaaaaaaaaaa",
|
||||
b"bbbbbbbbbbbbbbbb",
|
||||
@ -96,44 +97,6 @@ async def test_chk_capability(reactor, request, alice, params_idx, convergence_i
|
||||
assert actual == expected
|
||||
|
||||
|
||||
α = TypeVar("α")
|
||||
β = TypeVar("β")
|
||||
|
||||
async def asyncfoldr(
|
||||
i: Iterator[Awaitable[α]],
|
||||
f: Callable[[α, β], β],
|
||||
initial: β,
|
||||
) -> β:
|
||||
"""
|
||||
Right fold over an async iterator.
|
||||
|
||||
:param i: The async iterator.
|
||||
:param f: The function to fold.
|
||||
:param initial: The starting value.
|
||||
|
||||
:return: The result of the fold.
|
||||
"""
|
||||
result = initial
|
||||
async for a in i:
|
||||
result = f(a, result)
|
||||
return result
|
||||
|
||||
|
||||
def insert(item: tuple[α, β], d: dict[α, β]) -> dict[α, β]:
|
||||
"""
|
||||
In-place add an item to a dictionary.
|
||||
|
||||
If the key is already present, replace the value.
|
||||
|
||||
:param item: A tuple of the key and value.
|
||||
:param d: The dictionary to modify.
|
||||
|
||||
:return: The dictionary.
|
||||
"""
|
||||
d[item[0]] = item[1]
|
||||
return d
|
||||
|
||||
|
||||
@ensureDeferred
|
||||
async def skiptest_generate(reactor, request, alice):
|
||||
"""
|
||||
@ -154,36 +117,6 @@ async def skiptest_generate(reactor, request, alice):
|
||||
f.write(safe_dump(results))
|
||||
|
||||
|
||||
async def reconfigure(reactor, request, alice: TahoeProcess, params: tuple[int, int], convergence: bytes) -> None:
|
||||
"""
|
||||
Reconfigure a Tahoe-LAFS node with different ZFEC parameters and
|
||||
convergence secret.
|
||||
|
||||
:param reactor: A reactor to use to restart the process.
|
||||
:param request: The pytest request object to use to arrange process
|
||||
cleanup.
|
||||
:param alice: The Tahoe-LAFS node to reconfigure.
|
||||
:param params: The ``needed`` and ``total`` ZFEC encoding parameters.
|
||||
:param convergence: The convergence secret.
|
||||
|
||||
:return: ``None`` after the node configuration has been rewritten, the
|
||||
node has been restarted, and the node is ready to provide service.
|
||||
"""
|
||||
needed, total = params
|
||||
config = read_config(alice.node_dir, "tub.port")
|
||||
config.set_config("client", "shares.happy", str(1))
|
||||
config.set_config("client", "shares.needed", str(needed))
|
||||
config.set_config("client", "shares.total", str(total))
|
||||
config.write_private_config("convergence", base32.b2a(convergence))
|
||||
|
||||
# restart alice
|
||||
print(f"Restarting {alice.node_dir} for ZFEC reconfiguration")
|
||||
await alice.restart_async(reactor, request)
|
||||
print("Restarted. Waiting for ready state.")
|
||||
await_client_ready(alice)
|
||||
print("Ready.")
|
||||
|
||||
|
||||
async def generate(reactor, request, alice: TahoeProcess) -> AsyncGenerator[tuple[str, str], None]:
|
||||
"""
|
||||
Generate all of the test vectors using the given node.
|
||||
@ -220,22 +153,4 @@ def key(params: tuple[int, int], secret: bytes, data: bytes) -> str:
|
||||
:return: A distinct string for the given inputs, but shorter. This is
|
||||
suitable for use as, eg, a key in a dictionary.
|
||||
"""
|
||||
return f"{params[0]}/{params[1]},{digest(secret)},{digest(data)}"
|
||||
|
||||
|
||||
def upload(alice: TahoeProcess, fmt: str, data: bytes) -> str:
|
||||
"""
|
||||
Upload the given data to the given node.
|
||||
|
||||
:param alice: The node to upload to.
|
||||
|
||||
:param fmt: The name of the format for the upload. CHK, SDMF, or MDMF.
|
||||
|
||||
:param data: The data to upload.
|
||||
|
||||
:return: The capability for the uploaded data.
|
||||
"""
|
||||
with NamedTemporaryFile() as f:
|
||||
f.write(data)
|
||||
f.flush()
|
||||
return cli(alice, "put", f"--format={fmt}", f.name).decode("utf-8").strip()
|
||||
return f"{params[0]}/{params[1]},{hexdigest(secret)},{hexdigest(data)}"
|
||||
|
||||
@ -1,15 +1,9 @@
|
||||
"""
|
||||
Ported to Python 3.
|
||||
General functionality useful for the implementation of integration tests.
|
||||
"""
|
||||
from __future__ import unicode_literals
|
||||
from __future__ import absolute_import
|
||||
from __future__ import division
|
||||
from __future__ import print_function
|
||||
|
||||
from future.utils import PY2
|
||||
if PY2:
|
||||
from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401
|
||||
|
||||
from typing import TypeVar, Iterator, Awaitable, Callable
|
||||
from tempfile import NamedTemporaryFile
|
||||
import sys
|
||||
import time
|
||||
import json
|
||||
@ -32,6 +26,7 @@ import requests
|
||||
from paramiko.rsakey import RSAKey
|
||||
from boltons.funcutils import wraps
|
||||
|
||||
from allmydata.util import base32
|
||||
from allmydata.util.configutil import (
|
||||
get_config,
|
||||
set_config,
|
||||
@ -598,3 +593,89 @@ def run_in_thread(f):
|
||||
def test(*args, **kwargs):
|
||||
return deferToThread(lambda: f(*args, **kwargs))
|
||||
return test
|
||||
|
||||
|
||||
def upload(alice: TahoeProcess, fmt: str, data: bytes) -> str:
|
||||
"""
|
||||
Upload the given data to the given node.
|
||||
|
||||
:param alice: The node to upload to.
|
||||
|
||||
:param fmt: The name of the format for the upload. CHK, SDMF, or MDMF.
|
||||
|
||||
:param data: The data to upload.
|
||||
|
||||
:return: The capability for the uploaded data.
|
||||
"""
|
||||
with NamedTemporaryFile() as f:
|
||||
f.write(data)
|
||||
f.flush()
|
||||
return cli(alice, "put", f"--format={fmt}", f.name).decode("utf-8").strip()
|
||||
|
||||
|
||||
α = TypeVar("α")
|
||||
β = TypeVar("β")
|
||||
|
||||
async def asyncfoldr(
|
||||
i: Iterator[Awaitable[α]],
|
||||
f: Callable[[α, β], β],
|
||||
initial: β,
|
||||
) -> β:
|
||||
"""
|
||||
Right fold over an async iterator.
|
||||
|
||||
:param i: The async iterator.
|
||||
:param f: The function to fold.
|
||||
:param initial: The starting value.
|
||||
|
||||
:return: The result of the fold.
|
||||
"""
|
||||
result = initial
|
||||
async for a in i:
|
||||
result = f(a, result)
|
||||
return result
|
||||
|
||||
|
||||
def insert(item: tuple[α, β], d: dict[α, β]) -> dict[α, β]:
|
||||
"""
|
||||
In-place add an item to a dictionary.
|
||||
|
||||
If the key is already present, replace the value.
|
||||
|
||||
:param item: A tuple of the key and value.
|
||||
:param d: The dictionary to modify.
|
||||
|
||||
:return: The dictionary.
|
||||
"""
|
||||
d[item[0]] = item[1]
|
||||
return d
|
||||
|
||||
|
||||
async def reconfigure(reactor, request, node: TahoeProcess, params: tuple[int, int], convergence: bytes) -> None:
|
||||
"""
|
||||
Reconfigure a Tahoe-LAFS node with different ZFEC parameters and
|
||||
convergence secret.
|
||||
|
||||
:param reactor: A reactor to use to restart the process.
|
||||
:param request: The pytest request object to use to arrange process
|
||||
cleanup.
|
||||
:param node: The Tahoe-LAFS node to reconfigure.
|
||||
:param params: The ``needed`` and ``total`` ZFEC encoding parameters.
|
||||
:param convergence: The convergence secret.
|
||||
|
||||
:return: ``None`` after the node configuration has been rewritten, the
|
||||
node has been restarted, and the node is ready to provide service.
|
||||
"""
|
||||
needed, total = params
|
||||
config = node.get_config()
|
||||
config.set_config("client", "shares.happy", str(1))
|
||||
config.set_config("client", "shares.needed", str(needed))
|
||||
config.set_config("client", "shares.total", str(total))
|
||||
config.write_private_config("convergence", base32.b2a(convergence))
|
||||
|
||||
# restart the node
|
||||
print(f"Restarting {node.node_dir} for ZFEC reconfiguration")
|
||||
await node.restart_async(reactor, request)
|
||||
print("Restarted. Waiting for ready state.")
|
||||
await_client_ready(node)
|
||||
print("Ready.")
|
||||
|
||||
Loading…
Reference in New Issue
Block a user