From 8a427203019dc1ba2da1f0d3c1d0f589e90612a1 Mon Sep 17 00:00:00 2001 From: Jean-Paul Calderone Date: Thu, 22 Dec 2022 17:02:42 -0500 Subject: [PATCH] Move some general utility functions into the util module --- integration/test_vectors.py | 103 ++++-------------------------------- integration/util.py | 99 ++++++++++++++++++++++++++++++---- 2 files changed, 99 insertions(+), 103 deletions(-) diff --git a/integration/test_vectors.py b/integration/test_vectors.py index 7b939f5a7..042421e6f 100644 --- a/integration/test_vectors.py +++ b/integration/test_vectors.py @@ -4,9 +4,7 @@ Verify certain results against test vectors with well-known results. from __future__ import annotations -from typing import TypeVar, Iterator, Awaitable, Callable - -from tempfile import NamedTemporaryFile +from typing import AsyncGenerator from hashlib import sha256 from itertools import product from yaml import safe_dump @@ -15,13 +13,16 @@ from pytest import mark from pytest_twisted import ensureDeferred from . import vectors -from .util import cli, await_client_ready -from allmydata.client import read_config -from allmydata.util import base32 +from .util import reconfigure, upload, asyncfoldr, insert, TahoeProcess -def digest(bs: bytes) -> str: +def digest(bs: bytes) -> bytes: + return sha256(bs).digest() + + +def hexdigest(bs: bytes) -> str: return sha256(bs).hexdigest() + CONVERGENCE_SECRETS = [ b"aaaaaaaaaaaaaaaa", b"bbbbbbbbbbbbbbbb", @@ -96,44 +97,6 @@ async def test_chk_capability(reactor, request, alice, params_idx, convergence_i assert actual == expected -α = TypeVar("α") -β = TypeVar("β") - -async def asyncfoldr( - i: Iterator[Awaitable[α]], - f: Callable[[α, β], β], - initial: β, -) -> β: - """ - Right fold over an async iterator. - - :param i: The async iterator. - :param f: The function to fold. - :param initial: The starting value. - - :return: The result of the fold. - """ - result = initial - async for a in i: - result = f(a, result) - return result - - -def insert(item: tuple[α, β], d: dict[α, β]) -> dict[α, β]: - """ - In-place add an item to a dictionary. - - If the key is already present, replace the value. - - :param item: A tuple of the key and value. - :param d: The dictionary to modify. - - :return: The dictionary. - """ - d[item[0]] = item[1] - return d - - @ensureDeferred async def skiptest_generate(reactor, request, alice): """ @@ -154,36 +117,6 @@ async def skiptest_generate(reactor, request, alice): f.write(safe_dump(results)) -async def reconfigure(reactor, request, alice: TahoeProcess, params: tuple[int, int], convergence: bytes) -> None: - """ - Reconfigure a Tahoe-LAFS node with different ZFEC parameters and - convergence secret. - - :param reactor: A reactor to use to restart the process. - :param request: The pytest request object to use to arrange process - cleanup. - :param alice: The Tahoe-LAFS node to reconfigure. - :param params: The ``needed`` and ``total`` ZFEC encoding parameters. - :param convergence: The convergence secret. - - :return: ``None`` after the node configuration has been rewritten, the - node has been restarted, and the node is ready to provide service. - """ - needed, total = params - config = read_config(alice.node_dir, "tub.port") - config.set_config("client", "shares.happy", str(1)) - config.set_config("client", "shares.needed", str(needed)) - config.set_config("client", "shares.total", str(total)) - config.write_private_config("convergence", base32.b2a(convergence)) - - # restart alice - print(f"Restarting {alice.node_dir} for ZFEC reconfiguration") - await alice.restart_async(reactor, request) - print("Restarted. Waiting for ready state.") - await_client_ready(alice) - print("Ready.") - - async def generate(reactor, request, alice: TahoeProcess) -> AsyncGenerator[tuple[str, str], None]: """ Generate all of the test vectors using the given node. @@ -220,22 +153,4 @@ def key(params: tuple[int, int], secret: bytes, data: bytes) -> str: :return: A distinct string for the given inputs, but shorter. This is suitable for use as, eg, a key in a dictionary. """ - return f"{params[0]}/{params[1]},{digest(secret)},{digest(data)}" - - -def upload(alice: TahoeProcess, fmt: str, data: bytes) -> str: - """ - Upload the given data to the given node. - - :param alice: The node to upload to. - - :param fmt: The name of the format for the upload. CHK, SDMF, or MDMF. - - :param data: The data to upload. - - :return: The capability for the uploaded data. - """ - with NamedTemporaryFile() as f: - f.write(data) - f.flush() - return cli(alice, "put", f"--format={fmt}", f.name).decode("utf-8").strip() + return f"{params[0]}/{params[1]},{hexdigest(secret)},{hexdigest(data)}" diff --git a/integration/util.py b/integration/util.py index c2394375a..0dc63b2d5 100644 --- a/integration/util.py +++ b/integration/util.py @@ -1,15 +1,9 @@ """ -Ported to Python 3. +General functionality useful for the implementation of integration tests. """ -from __future__ import unicode_literals -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -from future.utils import PY2 -if PY2: - from future.builtins import filter, map, zip, ascii, chr, hex, input, next, oct, open, pow, round, super, bytes, dict, list, object, range, str, max, min # noqa: F401 +from typing import TypeVar, Iterator, Awaitable, Callable +from tempfile import NamedTemporaryFile import sys import time import json @@ -32,6 +26,7 @@ import requests from paramiko.rsakey import RSAKey from boltons.funcutils import wraps +from allmydata.util import base32 from allmydata.util.configutil import ( get_config, set_config, @@ -598,3 +593,89 @@ def run_in_thread(f): def test(*args, **kwargs): return deferToThread(lambda: f(*args, **kwargs)) return test + + +def upload(alice: TahoeProcess, fmt: str, data: bytes) -> str: + """ + Upload the given data to the given node. + + :param alice: The node to upload to. + + :param fmt: The name of the format for the upload. CHK, SDMF, or MDMF. + + :param data: The data to upload. + + :return: The capability for the uploaded data. + """ + with NamedTemporaryFile() as f: + f.write(data) + f.flush() + return cli(alice, "put", f"--format={fmt}", f.name).decode("utf-8").strip() + + +α = TypeVar("α") +β = TypeVar("β") + +async def asyncfoldr( + i: Iterator[Awaitable[α]], + f: Callable[[α, β], β], + initial: β, +) -> β: + """ + Right fold over an async iterator. + + :param i: The async iterator. + :param f: The function to fold. + :param initial: The starting value. + + :return: The result of the fold. + """ + result = initial + async for a in i: + result = f(a, result) + return result + + +def insert(item: tuple[α, β], d: dict[α, β]) -> dict[α, β]: + """ + In-place add an item to a dictionary. + + If the key is already present, replace the value. + + :param item: A tuple of the key and value. + :param d: The dictionary to modify. + + :return: The dictionary. + """ + d[item[0]] = item[1] + return d + + +async def reconfigure(reactor, request, node: TahoeProcess, params: tuple[int, int], convergence: bytes) -> None: + """ + Reconfigure a Tahoe-LAFS node with different ZFEC parameters and + convergence secret. + + :param reactor: A reactor to use to restart the process. + :param request: The pytest request object to use to arrange process + cleanup. + :param node: The Tahoe-LAFS node to reconfigure. + :param params: The ``needed`` and ``total`` ZFEC encoding parameters. + :param convergence: The convergence secret. + + :return: ``None`` after the node configuration has been rewritten, the + node has been restarted, and the node is ready to provide service. + """ + needed, total = params + config = node.get_config() + config.set_config("client", "shares.happy", str(1)) + config.set_config("client", "shares.needed", str(needed)) + config.set_config("client", "shares.total", str(total)) + config.write_private_config("convergence", base32.b2a(convergence)) + + # restart the node + print(f"Restarting {node.node_dir} for ZFEC reconfiguration") + await node.restart_async(reactor, request) + print("Restarted. Waiting for ready state.") + await_client_ready(node) + print("Ready.")